diff --git a/pytorch/05-clone-examples.sh b/pytorch/05-clone-examples.sh index a7531371..bbb3ee6a 100755 --- a/pytorch/05-clone-examples.sh +++ b/pytorch/05-clone-examples.sh @@ -1,4 +1,10 @@ #!/usr/bin/env bash set -e -git clone https://github.com/pytorch/examples.git examples + +if [ ! -d examples ]; then + git clone https://github.com/pytorch/examples.git examples +else + (cd examples && git pull) +fi + diff --git a/pytorch/06-run-example-mnist.sh b/pytorch/06-run-example-mnist.sh deleted file mode 100755 index 0e6809da..00000000 --- a/pytorch/06-run-example-mnist.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -source pytorch/.venv/bin/activate - -# TODO(#1144): Kill each of these. -# -# Pytorch tries to use and other GPUs leading to errors. -export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" - -EPOCHS="${EPOCHS:-5}" - -cd examples/mnist -python main.py --epochs "$EPOCHS" diff --git a/pytorch/06-run-examples.sh b/pytorch/06-run-examples.sh new file mode 100755 index 00000000..0ecd1f14 --- /dev/null +++ b/pytorch/06-run-examples.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +source pytorch/.venv/bin/activate + +# TODO(#1144): Kill each of these. +# +# Pytorch tries to use and other GPUs leading to errors. +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +# FIXME: This can hopefully be imminently deleted. Something weird is happening +# with dependencies: it seemed that this install call was required or else mnist +# complained about not being able to find pillow (PIL), but reading the log +# after adding this call, the dependency is already satisfied and the example +# happily runs. +python -m pip install pillow + +# Every example trains for this many epochs. +EPOCHS="${EPOCHS:-5}" + +echo "=== mnist ===" +( + cd examples/mnist + python main.py --epochs "$EPOCHS" +) + +echo "=== mnist_rnn ===" +( + cd examples/mnist_rnn + # Unlike most of the examples, the GPU is opt-in here, not opt-out. + python main.py --accel --epochs "$EPOCHS" +) + +echo "=== mnist_forward_forward ===" +( + cd examples/mnist_forward_forward + python main.py --epochs "$EPOCHS" +) + +echo "=== siamese_network ===" +( + cd examples/siamese_network + python main.py --epochs "$EPOCHS" +) + +# Train each language model type on the bundled wikitext-2 corpus, then +# generate text from the model.pt checkpoint that training leaves behind. +for model in RNN_TANH RNN_RELU LSTM GRU Transformer ; do + echo "=== word_language_model ($model) ===" + ( + cd examples/word_language_model + # The GPU is opt-in here too. + python main.py --accel --model "$model" --epochs "$EPOCHS" + python generate.py --accel + ) +done diff --git a/pytorch/07-run-datatypetests.sh b/pytorch/07-run-datatypetests.sh new file mode 100755 index 00000000..e4463059 --- /dev/null +++ b/pytorch/07-run-datatypetests.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script tests PyTorch datatypes by training two small models on MNIST data with +# three different precisions. We check the training curve follows "roughly the right +# shape" (defined via `EXPECTATIONS`). The script exits with 0 if and only if it runs +# to completion (with all precisions) and our criterion for the training curve +# following the right shape is satisfied. + + +# TODO(#1144): Kill. +# +# Pytorch tries to use and other GPUs leading to errors. +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + + +# PyTorch's addmm uses cuBLASLt, which SCALE does not support yet. Without this the +# TF32 runs fail with: +# RuntimeError: CUDA error: CUBLAS_STATUS_NOT_SUPPORTED when calling +# `cublasLtMatmulDescCreate(&raw_descriptor, compute_type, scale_type)` +# This makes PyTorch fall back to plain cuBLAS. +export DISABLE_ADDMM_CUDA_LT=1 + + +source pytorch/.venv/bin/activate + +python -u - <<'PY' +import contextlib +import os +import sys +import time + +import torch +from torch import nn +from torch.utils.data import DataLoader +from torchvision import datasets, transforms + +# Define the datatypes to use here +PRECISIONS = ("fp32", "tf32", "amp-bf16") + +EPOCHS = 5 +SEED = 0 + +# The per-epoch accuracy thresholds are a rising sequence: clearing all of +# them means the curve climbed with roughly the right shape. Calibrated on +# real GPU runs across every model x precision combination, then loosened to +# absorb non-determinism and toolchain differences. +EXPECTATIONS = { + "mlp": { + "max_initial_acc": 0.20, + "min_acc_at_epoch": {1: 0.90, 3: 0.94, 5: 0.95}, + "max_final_loss": 0.15, + }, + "cnn": { + "max_initial_acc": 0.20, + "min_acc_at_epoch": {1: 0.95, 3: 0.97, 5: 0.98}, + "max_final_loss": 0.10, + }, +} + +# How far accuracy may dip between checkpoints and still count as +# "non-decreasing" (epoch-to-epoch noise on a plateau). +MONOTONIC_TOL = 0.02 + +failures = [] + + +def check(ok, message): + if not ok: + failures.append(message) + print(f"FAIL: {message}") + + +def build_model(name): + if name == "mlp": + return nn.Sequential( + nn.Flatten(), + nn.Linear(28 * 28, 256), nn.ReLU(), + nn.Linear(256, 256), nn.ReLU(), + nn.Linear(256, 10), + ) + # A small LeNet-style CNN; the convolutions give broader kernel coverage. + return nn.Sequential( + nn.Conv2d(1, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), + nn.Flatten(), + nn.Linear(64 * 7 * 7, 128), nn.ReLU(), nn.Dropout(0.25), + nn.Linear(128, 10), + ) + + +def dataloaders(): + transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ) + train_set = datasets.MNIST("data", train=True, download=True, transform=transform) + test_set = datasets.MNIST("data", train=False, download=True, transform=transform) + return ( + DataLoader(train_set, batch_size=128, shuffle=True, pin_memory=True), + DataLoader(test_set, batch_size=512, pin_memory=True), + ) + + +def autocast(precision): + if precision == "amp-bf16": + return torch.autocast("cuda", dtype=torch.bfloat16) + return contextlib.nullcontext() + + +@torch.no_grad() +def evaluate(model, loader, precision): + model.eval() + criterion = nn.CrossEntropyLoss(reduction="sum") + total_loss, correct, total = 0.0, 0, 0 + for inputs, targets in loader: + inputs, targets = inputs.cuda(), targets.cuda() + with autocast(precision): + outputs = model(inputs) + total_loss += criterion(outputs, targets).item() + correct += (outputs.argmax(dim=1) == targets).sum().item() + total += targets.size(0) + return total_loss / total, correct / total + + +def train(model_name, precision, train_loader, test_loader): + """Return the untrained (loss, acc) and the per-epoch [(loss, acc)] curve.""" + + # Configure the backend + tf32 = precision == "tf32" + torch.backends.cuda.matmul.allow_tf32 = tf32 + torch.backends.cudnn.allow_tf32 = tf32 + torch.set_float32_matmul_precision("high" if tf32 else "highest") + + + torch.manual_seed(SEED) + torch.cuda.manual_seed_all(SEED) + + model = build_model(model_name).cuda() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + criterion = nn.CrossEntropyLoss() + + initial = evaluate(model, test_loader, precision) + print(f"[{model_name}/{precision}] epoch 0 loss={initial[0]:.4f} acc={initial[1]:.4f} (untrained)") + + curve = [] + for epoch in range(1, EPOCHS + 1): + model.train() + for inputs, targets in train_loader: + inputs, targets = inputs.cuda(), targets.cuda() + optimizer.zero_grad(set_to_none=True) + with autocast(precision): + loss = criterion(model(inputs), targets) + loss.backward() + optimizer.step() + test_loss, test_acc = evaluate(model, test_loader, precision) + curve.append((test_loss, test_acc)) + print(f"[{model_name}/{precision}] epoch {epoch} loss={test_loss:.4f} acc={test_acc:.4f}") + + return initial, curve + + +def check_curve(model_name, precision, initial, curve): + spec = EXPECTATIONS[model_name] + who = f"{model_name}/{precision}" + initial_loss, initial_acc = initial + acc_at = {epoch: curve[epoch - 1][1] for epoch in spec["min_acc_at_epoch"]} + final_loss = curve[-1][0] + + # The untrained network should score near chance (~10%). + check(initial_acc < spec["max_initial_acc"], + f"{who}: untrained accuracy {initial_acc:.4f} is too high to be a fresh network") + + # Accuracy must clear the rising thresholds and jump well above chance. + for epoch, threshold in sorted(spec["min_acc_at_epoch"].items()): + check(acc_at[epoch] >= threshold, + f"{who}: accuracy at epoch {epoch} was {acc_at[epoch]:.4f}, below {threshold:.2f}") + first_epoch = min(spec["min_acc_at_epoch"]) + check(acc_at[first_epoch] - initial_acc > 0.5, + f"{who}: accuracy barely moved ({initial_acc:.4f} -> {acc_at[first_epoch]:.4f}); " + "training is not learning") + + # Later checkpoints must not fall meaningfully below earlier ones. + epochs = sorted(spec["min_acc_at_epoch"]) + for earlier, later in zip(epochs, epochs[1:]): + check(acc_at[later] >= acc_at[earlier] - MONOTONIC_TOL, + f"{who}: accuracy fell from {acc_at[earlier]:.4f} (epoch {earlier}) " + f"to {acc_at[later]:.4f} (epoch {later})") + + # Test loss should fall well below its untrained value. + check(final_loss < initial_loss, + f"{who}: final loss {final_loss:.4f} did not improve on the untrained {initial_loss:.4f}") + check(final_loss < spec["max_final_loss"], + f"{who}: final loss {final_loss:.4f} exceeds the {spec['max_final_loss']:.2f} ceiling") + + +# Fail if CUDA is not available to prevent a silent fall-back to the CPU +if not torch.cuda.is_available(): + sys.exit("torch.cuda.is_available() is False; there is no GPU path to test.") +print(f"Device: {torch.cuda.get_device_name(0)}") + + +# Quick check that GPU kernels produce roughly the same result as CPU before we start training +torch.manual_seed(SEED) +a, b = torch.randn(256, 256), torch.randn(256, 256) +if not torch.allclose(a @ b, (a.cuda() @ b.cuda()).cpu(), atol=1e-3, rtol=1e-3): + sys.exit("GPU matmul does not match the CPU result.") + +# Constructing the datasets downloads MNIST on first use, so start the clock +# after this line to keep the (network-dependent) download out of the timing. +train_loader, test_loader = dataloaders() + +time_start = time.perf_counter() +for model_name in EXPECTATIONS: + for precision in PRECISIONS: + initial, curve = train(model_name, precision, train_loader, test_loader) + check_curve(model_name, precision, initial, curve) +elapsed = time.perf_counter() - time_start + +# Record the training+evaluation time in the build artifacts, whether or not +# the curve checks passed. +print(f"\nTotal training+evaluation time: {elapsed:.3f} seconds") +os.makedirs("/tmp/ci_benchmarks", exist_ok=True) +with open("/tmp/ci_benchmarks/datatypetests.txt", "w") as f: + f.write(f"training_seconds={elapsed:.3f}\n") + +if failures: + print(f"\n{len(failures)} check(s) failed:") + for message in failures: + print(f" - {message}") + sys.exit(1) +print("\nAll model/datatype training curves have the expected shape.") +PY diff --git a/pytorch/08-run-extendedtests.sh b/pytorch/08-run-extendedtests.sh new file mode 100755 index 00000000..80c2d967 --- /dev/null +++ b/pytorch/08-run-extendedtests.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -ETeuo pipefail + +# Keep PyTorch on the GPU selected by CI +# Fall back to the first GPU when CI has not already selected one +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +# Do not let packages from the runner's user site leak into the PyTorch venv +export PYTHONNOUSERSITE=1 + +# PyTorch's addmm uses cuBLASLt, which SCALE does not support yet. Without this the +# TF32 runs fail with: +# RuntimeError: CUDA error: CUBLAS_STATUS_NOT_SUPPORTED when calling +# `cublasLtMatmulDescCreate(&raw_descriptor, compute_type, scale_type)` +# This makes PyTorch fall back to plain cuBLAS. +export DISABLE_ADDMM_CUDA_LT=1 + +SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +SUITE_ROOT="${SCRIPT_DIR}/pytorch_extended_tests" +OUT_DIR="$(realpath .)" +SRCROOT="${OUT_DIR}/pytorch" +RESULTS_DIR="/tmp/ci_benchmarks/pytorch" + +if [[ ! -d "${SUITE_ROOT}" ]]; then + echo "Could not find the test suite in ${SUITE_ROOT}" + exit 1 +fi + +if [[ ! -d "${SRCROOT}" ]]; then + echo "Could not find the PyTorch source tree in ${SRCROOT}" + exit 1 +fi + +if [[ ! -f "${SRCROOT}/.venv/bin/activate" ]]; then + echo "Could not find .venv in ${SRCROOT}" + exit 1 +fi + +cd "${SRCROOT}" +source "${SRCROOT}/.venv/bin/activate" + +PYTHON="${PYTHON:-python}" + +if ! command -v "${PYTHON}" >/dev/null 2>&1; then + echo "Could not find Python executable: ${PYTHON}" + exit 1 +fi + +cd "${SUITE_ROOT}" + +# Start clean so the CI artefact only contains this run +rm -rf "${RESULTS_DIR}" +mkdir -p "${RESULTS_DIR}" + +# Keep both the src package and root config package importable +export PYTHONPATH="${SUITE_ROOT}/src:${SUITE_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" + +# Use unbuffered Python output so CI logs remain useful during a long run +export PYTHONUNBUFFERED=1 + +echo "Running pytorch_extended_tests" +echo "PyTorch source tree: ${SRCROOT}" +echo "Python: $(command -v "${PYTHON}")" +echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" +echo "Writing results to ${RESULTS_DIR}/" + +# Capture the full log but still return the suite's real exit status +set +e +"${PYTHON}" -u -m pytorch_extended_tests.orchestrator.run_suite \ + --results-dir "${RESULTS_DIR}" \ + --keep-existing \ + "$@" \ + |& tee "${RESULTS_DIR}/execution.log" +PIPE_STATUSES=("${PIPESTATUS[@]}") +set -e + +SUITE_STATUS="${PIPE_STATUSES[0]}" +TEE_STATUS="${PIPE_STATUSES[1]}" + +echo "Results are available in ${RESULTS_DIR}/" + +if [[ "${TEE_STATUS}" -ne 0 ]]; then + echo "Failed to write ${RESULTS_DIR}/execution.log" + exit "${TEE_STATUS}" +fi + +exit "${SUITE_STATUS}" diff --git a/pytorch/pytorch_extended_tests/.gitignore b/pytorch/pytorch_extended_tests/.gitignore new file mode 100644 index 00000000..3c122b93 --- /dev/null +++ b/pytorch/pytorch_extended_tests/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.venv/ +venv/ diff --git a/pytorch/pytorch_extended_tests/README.md b/pytorch/pytorch_extended_tests/README.md new file mode 100644 index 00000000..de2ce51e --- /dev/null +++ b/pytorch/pytorch_extended_tests/README.md @@ -0,0 +1,439 @@ +# pytorch_extended_tests + +A set of really extended PyTorch numerical tests which to run against different compiler and GPU builds + +The CI job runs the cases and saves raw outputs. It does not decide whether one build matches another. +For now, please somebody manually retrieve the CI artifacts and run the repeatability and cross-environment comparisons separately with the scripts in `manual_comparison_stuff/` + +That can be automated as well in future and put in CI somewhere, but doesn't fit into the way CI runs the scale validation repo. Should save reference standards somewhere, once there's enough accumulated. + +## Why the suite is split into levels + +The levels move from small, hopefully-easy-to-diagnose stuff towards longer pieces of work. If a training workload differs, looking back through the lower levels should help work out whether the first disagreement was in a basic tensor operation, a numerical kernel, autograd, mixed precision or the combined model itself +Examples different = bad. Look at lower levels to see where/why. Hopefully. + +- Level 0 runs four quick training and inference demonstrations: a linear classifier, MLP, CNN and attention model +- Level 1 does core tensor creation, arithmetic, mathematical functions, indexing, shape operations and type promotion +- Level 2 does reductions, matrix multiplication, convolution, pooling, linear solves, matrix factorisations, eigensystems, FFTs and special functions +- Level 3 does autograd graphs, neural-network layers, normalisation, attention, losses and optimiser updates +- Level 4 does float32 backend precision modes, FP16 and BF16 autocast, gradient scaling and serialisation round trips +- Level 5 combines the lower-level operations into short MLP, CNN and attention blocks with fixed optimiser updates +- Level 6 runs small tabular, image and text training workloads with fixed datasets and initial states + + +## How to use it + +For the default Level 0 run, generate the fixed model inputs once: + +```bash +python datasets/generate_datasets.py --only generated +``` + +Commit the generated `datasets/prepared/` files and the updated `datasets/dataset_manifest.json`. + +I want CI to only consume those prepared files, not regenerate data. Just in case that's a source of differences. But these should get moved to somewhere that CI can read from eventually, not stay in the scale validation repo xxx + +Level 0 does not need the externally downloaded datasets. +But we do need to download and prepare those from `datasets/README.md` before enabling Level 6 + +The normal CI entry point runs Level 0 only. Once that looks sensible, enable the other levels. This example runs the complete suite: + +```bash +cd .. +./run_pytorch_extended_tests.sh \ + --levels \ + level_0_smoke_workloads \ + level_1_core_tensor \ + level_2_numerical_kernels \ + level_3_autograd_and_learning \ + level_4_precision_and_execution \ + level_5_composite_models \ + level_6_real_workloads +``` + +Check the selected environment before running the suite: + +```bash +python tools/validate_setup.py +``` + +For the CPU reference job: + +```bash +python tools/validate_setup.py --device cpu +``` + +Run the normal CI entry point with: + +```bash +cd .. +./run_pytorch_extended_tests.sh +``` + +The script activates `pytorch/.venv`. Set `PYTHON` only when that environment needs a non-default interpreter command: + +```bash +cd .. +PYTHON=/path/to/python ./run_pytorch_extended_tests.sh +``` + +Set the CPU reference device with: + +```bash +cd .. +PYTORCH_EXTENDED_TESTS_DEVICE=cpu ./run_pytorch_extended_tests.sh +``` + +The shell script passes extra arguments to the Python orchestrator. For example, this runs only Level 4 FP32 cases on CPU: + +```bash +cd .. +PYTORCH_EXTENDED_TESTS_DEVICE=cpu \ + ./run_pytorch_extended_tests.sh \ + --levels level_4_precision_and_execution \ + --profiles controlled_fp32 +``` + +The Linux CI wrapper writes the complete raw bundle to: + +```text +/tmp/ci_benchmarks/pytorch +``` + +Check a completed result bundle with: + +```bash +python tools/inspect_result_bundle.py /tmp/ci_benchmarks/pytorch +``` + +To analyse repeatability, collect unmodified result bundles from repeated runs of the same environment beneath one directory. Each bundle directory must begin with `repeatability_`: + +```text +collected_runs/ +├── repeatability_run_001/ +├── repeatability_run_002/ +└── repeatability_run_003/ +``` + +Then run: + +```bash +python manual_comparison_stuff/analyse_repeatability.py collected_runs +``` + +This writes an overly detailed json file, a markdown report and graphs in `collected_runs/repeatability_analysis/`. Add `--write-populated-policy` when the runs are from the reference environment and the analyser will also fill the central policy template from the observed reference variability. + +To compare portable repeatability JSON files, put `reference.json` and the candidate json files in `repeatability_outputs/` and run `manual_comparison_stuff/compare_repeatability_analyses.py`. To perform the actual tensor-level GPU comparison, use `manual_comparison_stuff/compare_environment_outputs.py` with a `reference/` folder and one folder per compiler-GPU combo. + +## Precision profiles + +The names are slightly PyTorch-specific, but the basic idea is: + +- **FP32** is normal 32-bit floating point. The main baseline, because it has a hopefully sensible balance of speed, range and precision +- **FP16** is 16-bit floating point. It is faster and smaller on suitable GPUs, but has a much narrower numerical range and is easier to overflow or underflow +- **AMP** means Automatic Mixed Precision. PyTorch runs suitable operations in a lower precision while keeping numerically sensitive work and the main model state in FP32. AMP FP16 normally also uses gradient scaling to protect small gradients +- **BF16** is another 16-bit format. It has less precision than FP32 but a much wider range than FP16, so is apparently often easier to train with on newer hardware +- **FP64** is 64-bit floating point. It is mainly useful here as a high-precision diagnostic rather than a normal deep-learning setting... +- A `controlled_...` profile moves the inputs and model parameters themselves to that dtype and applies the suite's deterministic settings. An `amp_...` profile keeps the main state in FP32 and uses autocast for eligible operations + +I'd use FP32 and AMP FP16 only on CUDA for CI for now. That is the default in the config for now. FP32 gives the clearest baseline, while AMP FP16 tests the lower-precision path most likely to be used for normal GPU training without converting the model parameters themselves to raw FP16 + +The default profiles are: + +- CUDA: `controlled_fp32` and `amp_fp16` +- CPU: `controlled_fp32` + +`amp_fp16` keeps the model and optimiser state in FP32, runs eligible forward operations under FP16 autocast, and uses gradient scaling. This is a better initial test than `controlled_fp16`, which moves model parameters themselves to FP16 and is more likely to fail because of range or operator-support limitations + +The other profiles remain available as explicit opt-ins: + +- `controlled_fp64`: useful as a higher-precision diagnostic where the operation supports it +- `controlled_fp16`: raw FP16 tensors and model parameters on CUDA +- `controlled_bfloat16`: raw BF16 tensors and model parameters +- `amp_bfloat16`: FP32 model parameters with BF16 autocast + +For example, this adds BF16 autocast and FP64 to a CUDA run: + +```bash +cd .. +./run_pytorch_extended_tests.sh \ + --profiles controlled_fp32 amp_fp16 amp_bfloat16 controlled_fp64 +``` + +Do not add FP16 to the CPU reference job. CPU runs use FP32 by default; BF16 autocast can be enabled separately where the CPU and PyTorch build support it + +## Current scope and CUDA backend notes + +Currently tests one CPU or one GPU process at a time. It does **not** test multi-GPU stuff. + +Sparse tensors store only the non-zero parts of data which is mostly zero, using formats such as COO or CSR. They have their own storage invariants, operator coverage, autograd behaviour and CUDA kernels. I have left them out for now because the current suite is deliberately a dense-tensor baseline; adding a few sparse operations would probably just give an illusion of coverage without testing the important format and coalescing cases properly. Sparse support should be added later as a distinct category rather than mixed into the dense tests... + +Most CUDA maths in the suite will dispatch through the backend PyTorch selects, for example cuBLAS or cuBLASLt for matrix multiplication and cuFFT for FFTs. The repository tree marks test files containing cases which would normally use **cuDNN** on CUDA when cuDNN is available. Because no cuDNN yet. The marker does not mean that every case in that file uses cuDNN. + +The suite does not require `torch.backends.cudnn.is_available()` to be true. If PyTorch was built without cuDNN, convolution or normalisation operations will use a native CUDA implementation where PyTorch provides one. These paths can be slower and can produce different numerical results from cuDNN, which is useful to observe but means reference and candidate environments should have matching cuDNN availability when the aim is a like-for-like comparison. If a particular operation, dtype or shape has no fallback, the case fails and the exception is retained in the result bundle; it is not silently skipped. Maybe we should disable it for all of them for now actually rather than assuming that the installs mirror SCALE? Does our CUDA install have any packages that we don't do? + + + +## Repository layout + +```text +pytorch_extended_tests/ +├── README.md +├── manual_comparison_stuff/ +│ ├── README.md +│ ├── analyse_repeatability.py +│ ├── compare_environment_outputs.py +│ ├── compare_repeatability_analyses.py +│ ├── comparison_policy.py +│ ├── comparison_policy_template.json +│ └── level_0_first_look.py +├── cases/ +│ ├── README.md +│ ├── common/ +│ │ ├── README.md +│ │ ├── data.py +│ │ ├── demo.py +│ │ ├── dispatch.py +│ │ ├── learning.py +│ │ ├── mixed_precision.py +│ │ ├── models.py +│ │ ├── tensors.py +│ │ └── workloads.py +│ ├── level_0_smoke_workloads/ +│ │ ├── README.md +│ │ └── test_demo_workloads.py [cuDNN: CNN case] +│ ├── level_1_core_tensor/ +│ │ ├── test_tensor_creation_and_dtypes.py +│ │ ├── test_elementwise_arithmetic.py +│ │ ├── test_transcendental_functions.py +│ │ ├── test_indexing_and_shape.py +│ │ └── test_type_promotion.py +│ ├── level_2_numerical_kernels/ +│ │ ├── test_reductions_and_statistics.py +│ │ ├── test_matrix_multiplication.py +│ │ ├── test_convolution.py [cuDNN] +│ │ ├── test_pooling.py +│ │ ├── test_linear_solve.py +│ │ ├── test_factorisations.py +│ │ ├── test_eigensystems.py +│ │ ├── test_fft.py +│ │ └── test_special_functions.py +│ ├── level_3_autograd_and_learning/ +│ │ ├── test_autograd_elementwise.py +│ │ ├── test_autograd_matrix_ops.py [cuDNN: convolution case] +│ │ ├── test_nn_linear_and_conv.py [cuDNN: convolution cases] +│ │ ├── test_normalisation.py [cuDNN: BatchNorm cases where supported] +│ │ ├── test_attention.py +│ │ ├── test_losses.py +│ │ ├── test_optimizer_sgd.py +│ │ └── test_optimizer_adamw.py +│ ├── level_4_precision_and_execution/ +│ │ ├── README.md +│ │ ├── test_fp32_precision_modes.py [cuDNN: convolution case] +│ │ ├── test_amp_fp16.py +│ │ ├── test_amp_bfloat16.py +│ │ └── test_serialisation_roundtrip.py +│ ├── level_5_composite_models/ +│ │ ├── README.md +│ │ ├── test_mlp_block.py +│ │ ├── test_cnn_block.py [cuDNN] +│ │ └── test_attention_block.py +│ └── level_6_real_workloads/ +│ ├── README.md +│ ├── test_tabular_training_workload.py +│ ├── test_cnn_training_workload.py [cuDNN] +│ └── test_transformer_training_workload.py +├── ci/ +│ └── run_pytorch_extended_tests.ps1 +├── config/ +│ ├── README.md +│ ├── suite_config.py +│ └── test_catalogue.py +├── datasets/ +│ ├── README.md +│ ├── dataset_manifest.json +│ ├── generate_datasets.py +│ ├── downloaded/ +│ └── prepared/ +├── src/ +│ └── pytorch_extended_tests/ +│ ├── case_api.py +│ ├── precision_settings.py +│ ├── datasets/ +│ │ └── validation.py +│ ├── orchestrator/ +│ │ ├── execution_plan.py +│ │ ├── run_suite.py +│ │ ├── run_test_file.py +│ │ └── subprocess_runner.py +│ └── results/ +│ ├── artifact_writer.py +│ ├── level_0_summary.py +│ ├── observation.py +│ ├── result_bundle.py +│ └── tensor_storage.py +└── tools/ + ├── README.md + ├── inspect_result_bundle.py + └── validate_setup.py +``` + +Generated and downloaded dataset files are not all shown in the tree because that would make it fairly unreadable + +## What the non-level files are for + +- `README.md`: the main entry point for the repository + This file =) + +- `../run_pytorch_extended_tests.sh`: the Linux CI wrapper + It clears the fixed result directory, launches the Python orchestrator and keeps a combined execution log + The orchestrator performs configuration and dataset preflight checks before starting case processes + +- `manual_comparison_stuff/`: the manual repeatability and comparison harness + It is the only copy of these scripts in the repository; CI does not import or run them + +- `config/suite_config.py`: the central place for suite-wide choices + It holds the root seed, profiles, precision controls, timeouts, model sizes, optimiser settings and the enabled levels so these decisions are not copied into individual cases + +- `config/test_catalogue.py`: the stable map of test IDs, case IDs and output IDs + The orchestrator uses it to plan work and the manual comparison harness uses the same names to line up outputs from different builds + +- `config/README.md`: notes on changing central configuration + It calls out which changes require data regeneration or a version update and documents the environment-variable overrides + +- `datasets/generate_datasets.py`: the one manual data-generation and preprocessing script + It generates canonical numerical inputs, fixed model states and prepared versions of the downloaded datasets from the root seed + +- `datasets/dataset_manifest.json`: the record of dataset sources and generated files + It stores URLs, checksums, shapes and preprocessing metadata so CI can prove that each build used the same inputs + +- `datasets/README.md`: the dataset setup guide + It lists the download links, expected filenames, licences and the one-off preparation command + +- `cases/README.md`: the case-writing contract + It explains that case files produce raw named observations and must not contain comparison tolerances + +- `cases/common/data.py`: the prepared-array loader + It returns independent NumPy copies so an in-place test cannot modify the input seen by the next case + +- `cases/common/dispatch.py`: the small case-ID dispatcher + It keeps each test module's public `run_case` function consistent and gives a clear error for an unimplemented catalogue case + +- `cases/common/demo.py`: the small Level 0 summary builder + It turns the detailed block outputs into a few readable losses, predictions, logit statistics, activation statistics, gradient norms and parameter norms + +- `cases/common/tensors.py`: tensor conversion and structure helpers + It handles device and dtype conversion in one place and keeps complex and non-contiguous cases predictable + +- `cases/common/models.py`: the fixed shared model definitions + The parameter names match the generated initial-state files, which lets different builds start from exactly the same values + +- `cases/common/learning.py`: model, gradient and optimiser-state helpers + It snapshots named tensors in a stable way so training-related cases retain enough detail to locate the first divergence + +- `cases/common/mixed_precision.py`: the shared AMP and GradScaler helpers + It keeps scaler construction and step-skipping records consistent between the mixed-precision and composite-model cases + +- `cases/common/composite.py`: the shared short-training loop for Levels 0 and 5 + It records the same initial forward pass, first gradients, parameter checkpoints and evaluation outputs for each composite model + +- `cases/common/workloads.py`: the shared Level 6 training and evaluation loop + It fixes the batch order from the root seed and records the same losses, gradients, checkpoints, optimiser state and final metrics for all three real workloads + +- `src/pytorch_extended_tests/case_api.py`: the public interface passed to case modules + It exposes the selected profile, device, seed, temporary directory, prepared dataset paths and the output recorder protocol + +- `src/pytorch_extended_tests/precision_settings.py`: the compatibility layer for float32 precision controls + It avoids mixing old and new PyTorch TF32 APIs while still supporting older builds where cuDNN only exposes the legacy flag + +- `src/pytorch_extended_tests/datasets/validation.py`: the CI dataset preflight + It verifies required prepared files and checksums before any test process starts, so missing or stale inputs fail clearly + +- `src/pytorch_extended_tests/orchestrator/execution_plan.py`: the ordered task planner + It combines selected levels, tests, profiles and device into isolated test-module/profile tasks + +- `src/pytorch_extended_tests/orchestrator/run_suite.py`: the main Python suite runner + It validates data, runs the task plan, gathers statuses and finalises the raw result bundle + +- `src/pytorch_extended_tests/orchestrator/run_test_file.py`: the child-process entry point + It applies the profile, seeds the process, imports one test module and checks that every required output was produced + +- `src/pytorch_extended_tests/orchestrator/subprocess_runner.py`: the process-isolation wrapper + It sets the deterministic child environment, captures logs and protects the rest of the run from timeouts or CUDA failures in one module + +- `src/pytorch_extended_tests/results/level_0_summary.py`: the Level 0 CSV writer + It collects the required summary observation from each quick example and writes one row per example and precision profile + +- `src/pytorch_extended_tests/results/observation.py`: the JSON record model + It defines the stable machine-readable shape used for case statuses and named outputs + +- `src/pytorch_extended_tests/results/tensor_storage.py`: the lossless tensor binary format + It stores dtype, shape and raw bytes without silently converting lower-precision, complex or integer tensors + +- `src/pytorch_extended_tests/results/artifact_writer.py`: the observation and artifact writer + It validates output kinds, writes tensors into the artifact tree and records checksums and paths in JSONL + +- `src/pytorch_extended_tests/results/result_bundle.py`: the top-level bundle finaliser + It writes the manifest, merges task observations and creates the final execution summary even when some tasks fail + +- `tools/validate_setup.py`: the local and CI preflight command + It checks configuration, catalogue entries, datasets, selected profiles, PyTorch import, device availability and case-module imports + +- `manual_comparison_stuff/analyse_repeatability.py`: the repeated-run analysis tool + It measures within-environment variation, writes JSON/Markdown/graphs and can populate a central policy from the reference runs + +- `manual_comparison_stuff/comparison_policy_template.json`: the central comparison-policy starting point + It contains exact-match rules, static numerical floors and hard ceilings; reference repeatability fills the per-output limits + +- `manual_comparison_stuff/comparison_policy.py`: the shared policy calibration and judgement code + The manual comparison scripts use this module so policy population and PASS/MAYBE/FAIL decisions stay consistent + +- `manual_comparison_stuff/compare_repeatability_analyses.py`: the portable repeatability-summary comparator + It compares `reference.json` with other repeatability-analysis JSON files but cannot measure changed tensor values without the raw artefacts + +- `manual_comparison_stuff/compare_environment_outputs.py`: the full raw-output comparator + It calibrates the policy from `reference/`, checks each environment's repeatability and compares candidate runs with the reference tensor by tensor + +- `manual_comparison_stuff/level_0_first_look.py`: the rough manual Level 0 comparison script + It collates summary CSV files and gives a deliberately coarse PASS, FAIL or MAYBE result against `reference.csv` + +- `tools/inspect_result_bundle.py`: the completed-bundle checker + It parses the JSON files, verifies every tensor artifact and flags missing, corrupt or unreferenced files before results are archived + +- `tools/README.md`: quick notes for the maintenance tools + It gives the normal commands without making the root README even longer + +- `.gitignore`: exclusions for local Python and editor noise + Prepared test inputs are intentionally not ignored because they are part of the fixed inputs used by CI + +- `__init__.py` files: package markers and small public re-exports + They keep imports predictable without containing suite policy or test behaviour + +## Result files + +A successful Linux or CI invocation writes: + +```text +/tmp/ci_benchmarks/pytorch/ +├── run_manifest.json +├── test_catalogue.json +├── test_status.json +├── observations.jsonl +├── level_0_summary.csv +├── execution.log +└── artifacts/ +``` + + +The tensor values are stored as lossless binary artifacts. `observations.jsonl` contains their dtypes, shapes, checksums and relative paths + +`level_0_summary.csv` is written whenever Level 0 is selected. It is only a convenient first look; the detailed artefacts should be the gold standard comparison source of truth + +## Configuration notes + +Everything that is expected to stay consistent between builds should be centralised in `config/suite_config.py`. This includes the seed, backend profiles, AMP scaler settings, model dimensions, optimiser values and checkpoint choices + +Stable test, case and output IDs live in `config/test_catalogue.py` + +Numerical tolerances live in `manual_comparison_stuff/comparison_policy_template.json` and are populated from the reference repeatability analysis. + diff --git a/pytorch/pytorch_extended_tests/cases/README.md b/pytorch/pytorch_extended_tests/cases/README.md new file mode 100644 index 00000000..ab006529 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/README.md @@ -0,0 +1,23 @@ +# Case modules + +This folder contains the executable examples whose raw outputs are retained by CI + +Each test file maps directly to one entry in `config/test_catalogue.py`. The files produce named observations but do not decide whether those observations are numerically close enough to another build + +## Current implementation status + +- Level 0 quick model demonstrations are implemented and enabled by default +- Level 1 core tensor behaviour is implemented +- Level 2 numerical kernels is implemented +- Level 3 autograd and learning components is implemented +- Level 4 precision modes, mixed precision and serialisation is implemented +- Level 5 composite MLP, CNN and attention models are implemented +- Level 6 tabular, image and Transformer workloads are implemented + +All levels are implemented. The default enabled-level list contains Level 0 only so a new build gets a quick first check before the full suite is enabled + +## Shared helpers + +The `common/` folder contains the prepared-data loader, tensor conversion helpers, fixed model builders, learning-state helpers, mixed-precision helpers, the shared composite-model and workload loops and the small case dispatcher + +Case files should keep global choices in `config/suite_config.py` rather than adding local seeds, sizes, learning rates, scaler values or execution settings diff --git a/pytorch/pytorch_extended_tests/cases/__init__.py b/pytorch/pytorch_extended_tests/cases/__init__.py new file mode 100644 index 00000000..72465951 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/__init__.py @@ -0,0 +1 @@ +"""Executable case modules for the pytorch_extended_tests suite.""" diff --git a/pytorch/pytorch_extended_tests/cases/common/README.md b/pytorch/pytorch_extended_tests/cases/common/README.md new file mode 100644 index 00000000..bc6da38b --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/README.md @@ -0,0 +1,15 @@ +# Shared case helpers + +These files keep repeated setup and output handling out of the actual test cases + +- `data.py` loads independent copies of prepared NumPy arrays +- `demo.py` builds the small readable summary used by Level 0 and its CSV +- `dispatch.py` maps catalogue case IDs to their implementation functions +- `tensors.py` handles predictable device and dtype conversion +- `models.py` defines the fixed linear, MLP, CNN, attention and Transformer models used by generated initial states +- `learning.py` loads model state and snapshots parameters, gradients and optimiser internals +- `mixed_precision.py` builds the fixed AMP batch and records GradScaler decisions consistently +- `composite.py` runs the common two-step Level 5 optimisation path and records matching checkpoints for each model +- `workloads.py` runs the fixed Level 6 training and evaluation path, including exact batch rows, full checkpoint logits and optimiser state + +Suite-wide choices still belong in `config/suite_config.py`. These helpers should implement behaviour, not invent new policy diff --git a/pytorch/pytorch_extended_tests/cases/common/__init__.py b/pytorch/pytorch_extended_tests/cases/common/__init__.py new file mode 100644 index 00000000..70e182f3 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/__init__.py @@ -0,0 +1,64 @@ +"""Shared helpers used by the executable case modules.""" + +from cases.common.composite import run_composite_block +from cases.common.data import load_prepared_npz +from cases.common.demo import build_demo_summary +from cases.common.dispatch import run_registered_case +from cases.common.learning import ( + clone_module_state, + clone_named_gradients, + clone_named_parameters, + flatten_optimizer_state, + load_module_state, + module_to_profile, + tensor_mapping, +) +from cases.common.models import ( + build_attention_block, + build_linear_classifier, + build_cnn, + build_mlp, + build_sms_transformer, +) +from cases.common.mixed_precision import ( + build_mlp_batch, + make_grad_scaler, + parameters_changed, + scaler_state_record, +) +from cases.common.workloads import WorkloadBatch, run_training_workload +from cases.common.tensors import ( + as_profile_tensor, + describe_tensor, + describe_tensors, + paired_complex_dtype, +) + +__all__ = [ + "WorkloadBatch", + "as_profile_tensor", + "build_attention_block", + "build_demo_summary", + "build_linear_classifier", + "build_cnn", + "build_mlp", + "build_sms_transformer", + "build_mlp_batch", + "clone_module_state", + "clone_named_gradients", + "clone_named_parameters", + "describe_tensor", + "describe_tensors", + "flatten_optimizer_state", + "load_module_state", + "make_grad_scaler", + "load_prepared_npz", + "module_to_profile", + "paired_complex_dtype", + "parameters_changed", + "run_composite_block", + "run_registered_case", + "run_training_workload", + "scaler_state_record", + "tensor_mapping", +] diff --git a/pytorch/pytorch_extended_tests/cases/common/composite.py b/pytorch/pytorch_extended_tests/cases/common/composite.py new file mode 100644 index 00000000..916ee0ed --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/composite.py @@ -0,0 +1,162 @@ +"""Shared execution loop for the Level 0 and Level 5 composite models.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from config.suite_config import BLOCK_TESTS +from cases.common.learning import clone_named_gradients, clone_named_parameters +from cases.common.mixed_precision import make_grad_scaler +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +ForwardFunction = Callable[[Any, bool], tuple[Any, Mapping[str, Any]]] + + +def _build_optimizer( + model_name: str, + model: Any, + optimiser_settings: Mapping[str, Any] | None, +) -> Any: + import torch + + if optimiser_settings is None: + optimiser_name = str(BLOCK_TESTS["model_optimizers"][model_name]) + settings = BLOCK_TESTS[optimiser_name] + else: + optimiser_name = str(optimiser_settings["optimiser"]) + settings = optimiser_settings + + if optimiser_name == "sgd": + return torch.optim.SGD( + model.parameters(), + lr=float(settings["learning_rate"]), + momentum=float(settings.get("momentum", 0.0)), + weight_decay=float(settings.get("weight_decay", 0.0)), + ) + if optimiser_name == "adamw": + return torch.optim.AdamW( + model.parameters(), + lr=float(settings["learning_rate"]), + betas=tuple(float(value) for value in settings["betas"]), + eps=float(settings["epsilon"]), + weight_decay=float(settings.get("weight_decay", 0.0)), + ) + raise ValueError(f"Unknown composite-model optimiser: {optimiser_name}") + + +def _evaluation( + context: CaseContext, + model: Any, + labels: Any, + forward: ForwardFunction, + *, + retain_activations: bool, +) -> tuple[Any, float, dict[str, Any]]: + import torch + + was_training = model.training + model.eval() + with torch.no_grad(), context.autocast(): + logits, activations = forward(model, retain_activations) + loss = torch.nn.functional.cross_entropy(logits, labels) + model.train(was_training) + return ( + logits.detach().clone(), + float(loss.detach().cpu().item()), + {name: value.detach().clone() for name, value in activations.items()}, + ) + + +def run_composite_block( + context: CaseContext, + recorder: ObservationRecorder, + *, + model_name: str, + model: Any, + labels: Any, + forward: ForwardFunction, + optimiser_settings: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Run a fixed short optimisation path and retain each diagnostic checkpoint.""" + + import torch + + optimisation_steps = int(BLOCK_TESTS["optimisation_steps"]) + checkpoint_steps = tuple(int(value) for value in BLOCK_TESTS["checkpoint_steps"]) + optimizer = _build_optimizer(model_name, model, optimiser_settings) + scaler = make_grad_scaler(context) if context.profile_id == "amp_fp16" else None + + initial_logits, initial_loss, initial_activations = _evaluation( + context, + model, + labels, + forward, + retain_activations=True, + ) + initial_forward: dict[str, Any] = { + "logits": initial_logits, + "loss": torch.tensor(initial_loss, device=initial_logits.device, dtype=torch.float64), + } + initial_forward.update( + {f"activation.{name}": value for name, value in initial_activations.items()} + ) + + loss_series = [initial_loss] + parameter_states: dict[str, Any] = {"step_0": clone_named_parameters(model)} + evaluation_outputs: dict[str, Any] = {"step_0": {"logits": initial_logits}} + first_gradients: dict[str, Any] | None = None + + model.train() + for step in range(optimisation_steps): + optimizer.zero_grad(set_to_none=True) + with context.autocast(): + logits, _ = forward(model, False) + loss = torch.nn.functional.cross_entropy(logits, labels) + + if scaler is None: + loss.backward() + else: + scaler.scale(loss).backward() + scaler.unscale_(optimizer) + + if step == 0: + first_gradients = clone_named_gradients(model) + + if scaler is None: + optimizer.step() + else: + scaler.step(optimizer) + scaler.update() + + checkpoint = step + 1 + if checkpoint in checkpoint_steps: + checkpoint_logits, checkpoint_loss, _ = _evaluation( + context, + model, + labels, + forward, + retain_activations=False, + ) + loss_series.append(checkpoint_loss) + parameter_states[f"step_{checkpoint}"] = clone_named_parameters(model) + evaluation_outputs[f"step_{checkpoint}"] = {"logits": checkpoint_logits} + + if first_gradients is None: + raise RuntimeError("The composite block did not execute a backward pass") + if len(loss_series) != len(checkpoint_steps): + raise RuntimeError("Composite-model loss checkpoints do not match the configured steps") + + recorder.record("initial_forward", initial_forward) + recorder.record("loss_series", loss_series) + recorder.record("first_gradients", first_gradients) + recorder.record("parameter_states", parameter_states) + recorder.record("evaluation_outputs", evaluation_outputs) + + return { + "initial_forward": initial_forward, + "loss_series": loss_series, + "first_gradients": first_gradients, + "parameter_states": parameter_states, + "evaluation_outputs": evaluation_outputs, + } diff --git a/pytorch/pytorch_extended_tests/cases/common/data.py b/pytorch/pytorch_extended_tests/cases/common/data.py new file mode 100644 index 00000000..0ddacdbd --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/data.py @@ -0,0 +1,41 @@ +"""Load the canonical prepared arrays used by the case modules.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from pytorch_extended_tests.case_api import CaseContext + + +def load_prepared_npz( + context: CaseContext, + dataset_id: str, + filename: str, +) -> dict[str, np.ndarray]: + """Load one prepared NPZ file and return independent NumPy arrays.""" + + path = context.dataset_path(dataset_id) / filename + if not path.is_file(): + raise FileNotFoundError( + f"Prepared dataset file is missing: {path}\n" + "Run datasets/generate_datasets.py before running the suite" + ) + + try: + with np.load(path, allow_pickle=False) as archive: + # Copy these so a case can safely use an in-place operation + # The next case should still see the original prepared input + return {name: np.array(archive[name], copy=True) for name in archive.files} + except (OSError, ValueError) as exc: + raise RuntimeError(f"Could not load prepared dataset file: {path}") from exc + + +def require_prepared_file(context: CaseContext, dataset_id: str, filename: str) -> Path: + """Return one prepared file path after checking that it exists.""" + + path = context.dataset_path(dataset_id) / filename + if not path.is_file(): + raise FileNotFoundError(f"Prepared dataset file is missing: {path}") + return path diff --git a/pytorch/pytorch_extended_tests/cases/common/demo.py b/pytorch/pytorch_extended_tests/cases/common/demo.py new file mode 100644 index 00000000..05670b9a --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/demo.py @@ -0,0 +1,121 @@ +"""Build the concise human-facing summary for the Level 0 examples.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import Any + +from config.suite_config import LEVEL_0_DEMOS +from pytorch_extended_tests.case_api import CaseContext + + +def _tensor_l2(values: Mapping[str, Any]) -> float: + import torch + + total = torch.zeros((), dtype=torch.float64) + for value in values.values(): + if isinstance(value, Mapping): + nested = _tensor_l2(value) + total += nested * nested + elif isinstance(value, torch.Tensor): + current = value.detach().to(device="cpu", dtype=torch.float64) + total += torch.sum(current * current) + return math.sqrt(float(total.item())) + + +def _logit_stats(value: Any) -> dict[str, float]: + import torch + + current = value.detach().to(device="cpu", dtype=torch.float64) + return { + "mean": float(current.mean().item()), + "standard_deviation": float(current.std(unbiased=False).item()), + "maximum_absolute": float(current.abs().max().item()), + } + + +def build_demo_summary( + context: CaseContext, + *, + model_type: str, + optimiser_name: str, + labels: Any, + outputs: Mapping[str, Any], +) -> dict[str, Any]: + """Return the small set of values shown in level_0_summary.csv.""" + + import torch + + losses = [float(value) for value in outputs["loss_series"]] + initial_logits = outputs["initial_forward"]["logits"] + evaluation_outputs = outputs["evaluation_outputs"] + final_step = max(int(name.removeprefix("step_")) for name in evaluation_outputs) + final_logits = evaluation_outputs[f"step_{final_step}"]["logits"] + labels_cpu = labels.detach().to(device="cpu", dtype=torch.int64) + initial_predictions = torch.argmax(initial_logits.detach(), dim=1).to(device="cpu") + final_predictions = torch.argmax(final_logits.detach(), dim=1).to(device="cpu") + preview_count = int(LEVEL_0_DEMOS["prediction_preview_count"]) + + activations = { + name.removeprefix("activation."): value + for name, value in outputs["initial_forward"].items() + if name.startswith("activation.") + } + # Masked attention scores use the dtype minimum as a sentinel + # Keeping that sentinel in the quick aggregate makes the number useless + summary_activations = { + name: value + for name, value in activations.items() + if name != "attention_scores" + } + activation_names = sorted(summary_activations) + activation_values = [ + value.detach().to(device="cpu", dtype=torch.float64).reshape(-1) + for value in summary_activations.values() + ] + if activation_values: + combined_activations = torch.cat(activation_values) + activation_mean_absolute = float(combined_activations.abs().mean().item()) + activation_maximum_absolute = float(combined_activations.abs().max().item()) + else: + activation_mean_absolute = 0.0 + activation_maximum_absolute = 0.0 + + initial_stats = _logit_stats(initial_logits) + final_stats = _logit_stats(final_logits) + + return { + "example": context.case_id, + "model_type": model_type, + "optimiser": optimiser_name, + "training_steps": final_step, + "profile_id": context.profile_id, + "device": context.device, + "dtype": context.autocast_dtype_name or context.dtype_name, + "sample_count": int(labels_cpu.numel()), + "class_count": int(final_logits.shape[-1]), + "initial_loss": losses[0], + "final_loss": losses[-1], + "loss_change": losses[-1] - losses[0], + "initial_accuracy": float((initial_predictions == labels_cpu).float().mean().item()), + "final_accuracy": float((final_predictions == labels_cpu).float().mean().item()), + "prediction_changes": int((initial_predictions != final_predictions).sum().item()), + "initial_predictions": [int(value) for value in initial_predictions[:preview_count]], + "final_predictions": [int(value) for value in final_predictions[:preview_count]], + "initial_logits_mean": initial_stats["mean"], + "initial_logits_standard_deviation": initial_stats["standard_deviation"], + "initial_logits_maximum_absolute": initial_stats["maximum_absolute"], + "final_logits_mean": final_stats["mean"], + "final_logits_standard_deviation": final_stats["standard_deviation"], + "final_logits_maximum_absolute": final_stats["maximum_absolute"], + "first_gradient_l2": _tensor_l2(outputs["first_gradients"]), + "initial_parameter_l2": _tensor_l2(outputs["parameter_states"]["step_0"]), + "final_parameter_l2": _tensor_l2( + outputs["parameter_states"][f"step_{final_step}"] + ), + "activation_count": len(activation_names), + "activation_mean_absolute": activation_mean_absolute, + "activation_maximum_absolute": activation_maximum_absolute, + "activation_names": activation_names, + } diff --git a/pytorch/pytorch_extended_tests/cases/common/dispatch.py b/pytorch/pytorch_extended_tests/cases/common/dispatch.py new file mode 100644 index 00000000..3aee74f9 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/dispatch.py @@ -0,0 +1,29 @@ +"""Small dispatch helper shared by case files.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +CaseFunction = Callable[[CaseContext, ObservationRecorder], None] + + +def run_registered_case( + context: CaseContext, + recorder: ObservationRecorder, + cases: Mapping[str, CaseFunction], +) -> None: + """Run the function registered for the current catalogue case ID.""" + + try: + case_function = cases[context.case_id] + except KeyError as exc: + known = ", ".join(sorted(cases)) + raise KeyError( + f"No implementation is registered for {context.case_id!r}\n" + f"Known cases: {known}" + ) from exc + + case_function(context, recorder) diff --git a/pytorch/pytorch_extended_tests/cases/common/learning.py b/pytorch/pytorch_extended_tests/cases/common/learning.py new file mode 100644 index 00000000..0559faa0 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/learning.py @@ -0,0 +1,152 @@ +"""Helpers for loading fixed states and recording learning internals.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from cases.common.data import load_prepared_npz +from pytorch_extended_tests.case_api import CaseContext + + +MODEL_DATASET_ID = "model_inputs_v1" + + +def load_module_state(context: CaseContext, module: Any, filename: str) -> None: + """Load one generated state file into a module without changing its dtype.""" + + import torch + + arrays = load_prepared_npz(context, MODEL_DATASET_ID, filename) + expected = module.state_dict() + if set(arrays) != set(expected): + missing = sorted(set(expected) - set(arrays)) + unexpected = sorted(set(arrays) - set(expected)) + raise ValueError( + f"Initial state does not match the module\n" + f"Missing keys: {missing}\n" + f"Unexpected keys: {unexpected}" + ) + + converted: dict[str, Any] = {} + for name, target in expected.items(): + value = torch.from_numpy(arrays[name]) + converted[name] = value.to(device=target.device, dtype=target.dtype) + module.load_state_dict(converted, strict=True) + + +def clone_named_parameters(module: Any) -> dict[str, Any]: + """Clone all named parameters in state-dictionary order.""" + + return { + name: parameter.detach().clone() + for name, parameter in module.named_parameters() + } + + +def clone_named_gradients(module: Any) -> dict[str, Any]: + """Clone every parameter gradient and fail clearly when one is missing.""" + + gradients: dict[str, Any] = {} + for name, parameter in module.named_parameters(): + if parameter.grad is None: + raise RuntimeError(f"Parameter did not receive a gradient: {name}") + gradients[name] = parameter.grad.detach().clone() + return gradients + + +def clone_module_state(module: Any) -> dict[str, Any]: + """Clone parameters and buffers from a module state dictionary.""" + + return { + name: value.detach().clone() + for name, value in module.state_dict().items() + } + + +def flatten_optimizer_state(optimizer: Any, module: Any) -> dict[str, Any]: + """Return optimiser tensors with stable parameter names.""" + + import torch + + parameter_names = {parameter: name for name, parameter in module.named_parameters()} + first_parameter = next(module.parameters()) + output: dict[str, Any] = {} + + # Keep the main numeric group settings as tensors as plain SGD has no state tensors + # This also makes it obvious when two jobs used different optimiser settings + numeric_group_fields = ( + "lr", + "momentum", + "dampening", + "weight_decay", + "eps", + "maximize", + "nesterov", + "amsgrad", + ) + for group_index, group in enumerate(optimizer.param_groups): + prefix = f"param_group_{group_index}" + for field in numeric_group_fields: + value = group.get(field) + if isinstance(value, (bool, int, float)): + # Keep optimiser settings independent of the model precision + # Storing an LR in FP16 can round or underflow a configuration value + if isinstance(value, bool): + stored_value = int(value) + dtype = torch.int64 + elif isinstance(value, int): + stored_value = value + dtype = torch.int64 + else: + stored_value = value + dtype = torch.float64 + output[f"{prefix}.{field}"] = torch.tensor( + stored_value, + device=first_parameter.device, + dtype=dtype, + ) + betas = group.get("betas") + if isinstance(betas, tuple) and len(betas) == 2: + output[f"{prefix}.beta1"] = torch.tensor( + betas[0], device=first_parameter.device, dtype=torch.float64 + ) + output[f"{prefix}.beta2"] = torch.tensor( + betas[1], device=first_parameter.device, dtype=torch.float64 + ) + + for parameter, state in optimizer.state.items(): + parameter_name = parameter_names.get(parameter) + if parameter_name is None: + raise RuntimeError("Optimiser contains a parameter which is not in the model") + for state_name, value in state.items(): + if isinstance(value, torch.Tensor): + output[f"{parameter_name}.{state_name}"] = value.detach().clone() + elif isinstance(value, (bool, int, float)): + if isinstance(value, bool): + stored_value = int(value) + dtype = torch.int64 + elif isinstance(value, int): + stored_value = value + dtype = torch.int64 + else: + stored_value = value + dtype = torch.float64 + output[f"{parameter_name}.{state_name}"] = torch.tensor( + stored_value, + device=first_parameter.device, + dtype=dtype, + ) + return output + + +def module_to_profile(context: CaseContext, module: Any) -> Any: + """Move a module to the selected device and ordinary profile dtype.""" + + return module.to(device=context.device, dtype=context.torch_dtype()) + + +def tensor_mapping(values: Mapping[str, Any]) -> dict[str, Any]: + """Detach and clone a named tensor mapping.""" + + return {name: value.detach().clone() for name, value in values.items()} diff --git a/pytorch/pytorch_extended_tests/cases/common/mixed_precision.py b/pytorch/pytorch_extended_tests/cases/common/mixed_precision.py new file mode 100644 index 00000000..f2e209b6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/mixed_precision.py @@ -0,0 +1,96 @@ +"""Shared helpers for the mixed-precision cases.""" + +from __future__ import annotations + +from typing import Any + +from config.suite_config import AMP_GRAD_SCALER +from cases.common.data import load_prepared_npz +from cases.common.learning import ( + clone_named_gradients, + clone_named_parameters, + load_module_state, + module_to_profile, +) +from cases.common.models import build_mlp +from cases.common.tensors import as_profile_tensor +from pytorch_extended_tests.case_api import CaseContext + + +MODEL_DATASET_ID = "model_inputs_v1" + + +def build_mlp_batch(context: CaseContext) -> tuple[Any, Any, Any]: + """Build the fixed MLP and return its canonical input and labels.""" + + import torch + + arrays = load_prepared_npz(context, MODEL_DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["mlp_input"], requires_grad=True) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + model = module_to_profile(context, build_mlp()) + load_module_state(context, model, "mlp_initial_state.npz") + return model, value, labels + + +def make_grad_scaler(context: CaseContext) -> Any: + """Build the configured GradScaler using the public device-aware API.""" + + import torch + + settings = AMP_GRAD_SCALER + device_type = torch.device(context.device).type + try: + return torch.amp.GradScaler( + device_type, + init_scale=float(settings["initial_scale"]), + growth_factor=float(settings["growth_factor"]), + backoff_factor=float(settings["backoff_factor"]), + growth_interval=int(settings["growth_interval"]), + enabled=True, + ) + except TypeError: + # Older PyTorch releases exposed the CUDA scaler without a device argument + # Keep this fallback until all tested builds use the newer torch.amp API + return torch.cuda.amp.GradScaler( + init_scale=float(settings["initial_scale"]), + growth_factor=float(settings["growth_factor"]), + backoff_factor=float(settings["backoff_factor"]), + growth_interval=int(settings["growth_interval"]), + enabled=True, + ) + + +def parameters_changed(before: dict[str, Any], after: dict[str, Any]) -> bool: + """Return whether any named parameter changed exactly.""" + + import torch + + if before.keys() != after.keys(): + raise ValueError("Parameter mappings do not have the same keys") + return any(not torch.equal(before[name], after[name]) for name in before) + + +def scaler_state_record( + scaler: Any, + *, + initial_scale: float, + step_requested: bool, + step_skipped: bool, + overflow_injected: bool, +) -> dict[str, Any]: + """Return the public scaler state with the decisions made by this case.""" + + state = dict(scaler.state_dict()) + return { + "enabled": bool(scaler.is_enabled()), + "initial_scale": float(initial_scale), + "final_scale": float(scaler.get_scale()), + "growth_factor": float(state.get("growth_factor", 1.0)), + "backoff_factor": float(state.get("backoff_factor", 1.0)), + "growth_interval": int(state.get("growth_interval", 0)), + "growth_tracker": int(state.get("_growth_tracker", 0)), + "step_requested": bool(step_requested), + "step_skipped": bool(step_skipped), + "overflow_injected": bool(overflow_injected), + } diff --git a/pytorch/pytorch_extended_tests/cases/common/models.py b/pytorch/pytorch_extended_tests/cases/common/models.py new file mode 100644 index 00000000..831900ed --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/models.py @@ -0,0 +1,293 @@ +"""Small fixed models shared by the learning cases.""" + +from __future__ import annotations + +import math +from typing import Any + +from config.suite_config import MODEL_ARCHITECTURES + + +def build_linear_classifier() -> Any: + """Build the small linear classifier used by the Level 0 demo.""" + + import torch + + architecture = MODEL_ARCHITECTURES["linear"] + + class FixedLinearClassifier(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear = torch.nn.Linear( + int(architecture["input_features"]), + int(architecture["output_features"]), + ) + + def forward(self, value: Any, *, return_activations: bool = False) -> Any: + logits = self.linear(value) + if return_activations: + return logits, {"logits": logits} + return logits + + return FixedLinearClassifier() + + +def build_mlp() -> Any: + """Build the MLP whose parameter names match the generated initial state.""" + + import torch + + architecture = MODEL_ARCHITECTURES["mlp"] + + class FixedMLP(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + dimensions = ( + architecture["input_features"], + *architecture["hidden_features"], + architecture["output_features"], + ) + self.layers = torch.nn.ModuleList( + torch.nn.Linear(input_size, output_size) + for input_size, output_size in zip(dimensions, dimensions[1:]) + ) + + def forward(self, value: Any, *, return_activations: bool = False) -> Any: + activations: dict[str, Any] = {} + current = value + for index, layer in enumerate(self.layers): + current = layer(current) + activations[f"linear_{index}"] = current + if index + 1 != len(self.layers): + current = torch.relu(current) + activations[f"relu_{index}"] = current + if return_activations: + return current, activations + return current + + return FixedMLP() + + +def build_cnn() -> Any: + """Build the small CNN used by block and image-workload cases.""" + + import torch + + architecture = MODEL_ARCHITECTURES["cnn"] + input_channels, first_channels, second_channels = architecture["channels"] + + class FixedCNN(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.features = torch.nn.Sequential( + torch.nn.Conv2d(input_channels, first_channels, kernel_size=3, padding=1), + torch.nn.ReLU(), + torch.nn.MaxPool2d(kernel_size=2), + torch.nn.Conv2d(first_channels, second_channels, kernel_size=3, padding=1), + torch.nn.ReLU(), + torch.nn.MaxPool2d(kernel_size=2), + ) + flattened_features = second_channels * 7 * 7 + self.classifier = torch.nn.Sequential( + torch.nn.Linear( + flattened_features, + architecture["classifier_hidden_features"], + ), + torch.nn.ReLU(), + torch.nn.Linear( + architecture["classifier_hidden_features"], + architecture["classes"], + ), + ) + + def forward(self, value: Any, *, return_activations: bool = False) -> Any: + activations: dict[str, Any] = {} + current = value + activation_names = ( + "conv_0", + "relu_0", + "pool_0", + "conv_1", + "relu_1", + "pool_1", + ) + for name, layer in zip(activation_names, self.features): + current = layer(current) + activations[name] = current + + current = torch.flatten(current, start_dim=1) + activations["flattened"] = current + current = self.classifier[0](current) + activations["classifier_linear_0"] = current + current = self.classifier[1](current) + activations["classifier_relu_0"] = current + current = self.classifier[2](current) + activations["logits"] = current + + if return_activations: + return current, activations + return current + + return FixedCNN() + + +def build_attention_block() -> Any: + """Build a small residual multi-head attention classifier.""" + + import torch + + architecture = MODEL_ARCHITECTURES["attention"] + embedding_size = int(architecture["embedding_size"]) + head_count = int(architecture["heads"]) + head_size = embedding_size // head_count + + class FixedAttentionBlock(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.q_proj = torch.nn.Linear(embedding_size, embedding_size) + self.k_proj = torch.nn.Linear(embedding_size, embedding_size) + self.v_proj = torch.nn.Linear(embedding_size, embedding_size) + self.out_proj = torch.nn.Linear(embedding_size, embedding_size) + self.norm = torch.nn.LayerNorm(embedding_size) + self.classifier = torch.nn.Linear(embedding_size, architecture["classes"]) + + def _split_heads(self, value: Any) -> Any: + batch_size, sequence_length, _ = value.shape + return value.reshape( + batch_size, + sequence_length, + head_count, + head_size, + ).transpose(1, 2) + + def forward( + self, + value: Any, + padding_mask: Any, + *, + return_activations: bool = False, + ) -> Any: + query = self._split_heads(self.q_proj(value)) + key = self._split_heads(self.k_proj(value)) + projected_value = self._split_heads(self.v_proj(value)) + + scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(head_size) + key_mask = padding_mask[:, None, None, :] + scores = scores.masked_fill(key_mask, torch.finfo(scores.dtype).min) + weights = torch.softmax(scores, dim=-1) + attended = torch.matmul(weights, projected_value) + attended = attended.transpose(1, 2).contiguous().reshape(value.shape) + + projected = self.out_proj(attended) + normalised = self.norm(value + projected) + valid_tokens = (~padding_mask).to(dtype=normalised.dtype).unsqueeze(-1) + pooled = (normalised * valid_tokens).sum(dim=1) / valid_tokens.sum(dim=1) + logits = self.classifier(pooled) + + if return_activations: + return logits, { + "query": query, + "key": key, + "value": projected_value, + "attention_scores": scores, + "attention_weights": weights, + "attended": attended, + "projected": projected, + "normalised": normalised, + "pooled": pooled, + } + return logits + + return FixedAttentionBlock() + + +def build_sms_transformer() -> Any: + """Build the small Transformer used by the SMS workload.""" + + import torch + + architecture = MODEL_ARCHITECTURES["sms_transformer"] + sequence_length = int(architecture["sequence_length"]) + embedding_size = int(architecture["embedding_size"]) + vocabulary_size = int(architecture["vocabulary_size"]) + + class FixedSMSTransformer(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.token_embedding = torch.nn.Embedding( + vocabulary_size, + embedding_size, + padding_idx=0, + ) + self.position_embedding = torch.nn.Embedding( + sequence_length, + embedding_size, + ) + layer = torch.nn.TransformerEncoderLayer( + d_model=embedding_size, + nhead=int(architecture["heads"]), + dim_feedforward=int(architecture["feedforward_size"]), + dropout=float(architecture["dropout"]), + activation=str(architecture["activation"]), + batch_first=True, + norm_first=bool(architecture["norm_first"]), + ) + try: + self.encoder = torch.nn.TransformerEncoder( + layer, + num_layers=int(architecture["layers"]), + enable_nested_tensor=False, + ) + except TypeError: + # Older PyTorch releases do not expose the nested-tensor switch + # The ordinary padded path is still selected by the Boolean mask + self.encoder = torch.nn.TransformerEncoder( + layer, + num_layers=int(architecture["layers"]), + ) + self.final_norm = torch.nn.LayerNorm(embedding_size) + self.classifier = torch.nn.Linear( + embedding_size, + int(architecture["classes"]), + ) + + def forward( + self, + input_ids: Any, + attention_mask: Any, + *, + return_activations: bool = False, + ) -> Any: + batch_size, current_length = input_ids.shape + if current_length > sequence_length: + raise ValueError( + f"Input sequence length {current_length} exceeds {sequence_length}" + ) + + positions = torch.arange( + current_length, + device=input_ids.device, + dtype=torch.int64, + ).unsqueeze(0).expand(batch_size, -1) + embedded = self.token_embedding(input_ids) + self.position_embedding(positions) + padding_mask = ~attention_mask + encoded = self.encoder( + embedded, + src_key_padding_mask=padding_mask, + ) + normalised = self.final_norm(encoded) + valid_tokens = attention_mask.to(dtype=normalised.dtype).unsqueeze(-1) + pooled = (normalised * valid_tokens).sum(dim=1) / valid_tokens.sum(dim=1) + logits = self.classifier(pooled) + + if return_activations: + return logits, { + "embedded": embedded, + "encoded": encoded, + "normalised": normalised, + "pooled": pooled, + } + return logits + + return FixedSMSTransformer() + diff --git a/pytorch/pytorch_extended_tests/cases/common/tensors.py b/pytorch/pytorch_extended_tests/cases/common/tensors.py new file mode 100644 index 00000000..9cb41203 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/tensors.py @@ -0,0 +1,73 @@ +"""Tensor conversion and structure helpers for case outputs.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import numpy as np + +from pytorch_extended_tests.case_api import CaseContext + + +def paired_complex_dtype(dtype: Any) -> Any: + """Return the complex dtype corresponding to a real PyTorch dtype.""" + + import torch + + if dtype == torch.float64: + return torch.complex128 + return torch.complex64 + + +def as_profile_tensor( + context: CaseContext, + value: np.ndarray | Any, + *, + dtype: Any | None = None, + requires_grad: bool = False, +) -> Any: + """Move an array to the case device with predictable dtype handling.""" + + import torch + + array = np.asarray(value) + tensor = torch.from_numpy(np.ascontiguousarray(array)) + + if dtype is None: + if np.issubdtype(array.dtype, np.floating): + dtype = context.torch_dtype() + elif np.issubdtype(array.dtype, np.complexfloating): + dtype = paired_complex_dtype(context.torch_dtype()) + + tensor = tensor.to(device=context.device, dtype=dtype) + if requires_grad: + if not tensor.is_floating_point() and not tensor.is_complex(): + raise TypeError("Only floating-point and complex tensors can require gradients") + tensor.requires_grad_(True) + return tensor + + +def describe_tensor(value: Any) -> dict[str, Any]: + """Return comparison-friendly tensor structure without embedding values.""" + + import torch + + if not isinstance(value, torch.Tensor): + raise TypeError(f"Expected a PyTorch tensor, got {type(value)!r}") + + return { + "shape": list(value.shape), + "dtype": str(value.dtype).removeprefix("torch."), + "strides": list(value.stride()), + "layout": str(value.layout).removeprefix("torch."), + "is_contiguous": bool(value.is_contiguous()), + "numel": value.numel(), + "requires_grad": bool(value.requires_grad), + } + + +def describe_tensors(values: Mapping[str, Any]) -> dict[str, Any]: + """Describe a named tensor mapping in stable insertion order.""" + + return {name: describe_tensor(value) for name, value in values.items()} diff --git a/pytorch/pytorch_extended_tests/cases/common/workloads.py b/pytorch/pytorch_extended_tests/cases/common/workloads.py new file mode 100644 index 00000000..f27e46f6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/common/workloads.py @@ -0,0 +1,365 @@ +"""Shared training and evaluation loop for the Level 6 workloads.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +import numpy as np + +from config.suite_config import DATALOADER, OUTPUT_CAPTURE, WORKLOAD_CAPTURE, WORKLOADS +from cases.common.learning import ( + clone_named_gradients, + clone_named_parameters, + flatten_optimizer_state, +) +from cases.common.mixed_precision import make_grad_scaler +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + + +@dataclass(frozen=True, slots=True) +class WorkloadBatch: + """One model batch with positional inputs, keyword inputs and labels.""" + + args: tuple[Any, ...] + kwargs: Mapping[str, Any] + labels: Any + + +BatchBuilder = Callable[[np.ndarray], WorkloadBatch] +ForwardFunction = Callable[[Any, WorkloadBatch], Any] + + +def _build_optimizer(settings: Mapping[str, Any], model: Any) -> Any: + import torch + + optimiser_name = str(settings["optimiser"]) + if optimiser_name == "sgd": + return torch.optim.SGD( + model.parameters(), + lr=float(settings["learning_rate"]), + momentum=float(settings["momentum"]), + weight_decay=float(settings["weight_decay"]), + ) + if optimiser_name == "adamw": + return torch.optim.AdamW( + model.parameters(), + lr=float(settings["learning_rate"]), + betas=tuple(float(value) for value in settings["betas"]), + eps=float(settings["epsilon"]), + weight_decay=float(settings["weight_decay"]), + ) + raise ValueError(f"Unknown workload optimiser: {optimiser_name}") + + +def _batch_schedule( + *, + sample_count: int, + batch_size: int, + step_count: int, + seed: int, + shuffle: bool, + drop_last: bool, +) -> tuple[np.ndarray, ...]: + """Build the exact training rows used by every optimisation step.""" + + if sample_count < 1: + raise ValueError("The training dataset must contain at least one sample") + if batch_size < 1: + raise ValueError("The training batch size must be positive") + if drop_last and sample_count < batch_size: + raise ValueError("drop_last cannot be used when the dataset is smaller than one batch") + + generator = np.random.Generator(np.random.PCG64(seed)) + batches: list[np.ndarray] = [] + order = np.arange(sample_count, dtype=np.int64) + position = sample_count + + for _ in range(step_count): + if position >= sample_count or (drop_last and position + batch_size > sample_count): + order = ( + generator.permutation(sample_count).astype(np.int64, copy=False) + if shuffle + else np.arange(sample_count, dtype=np.int64) + ) + position = 0 + + stop = min(position + batch_size, sample_count) + batches.append(np.array(order[position:stop], dtype=np.int64, copy=True)) + position = stop + + return tuple(batches) + + +def _evaluation_rows(sample_count: int, batch_size: int) -> tuple[np.ndarray, ...]: + rows = np.arange(sample_count, dtype=np.int64) + return tuple( + np.array(rows[start : start + batch_size], copy=True) + for start in range(0, sample_count, batch_size) + ) + + +def _evaluate( + context: CaseContext, + model: Any, + *, + rows: tuple[np.ndarray, ...], + build_batch: BatchBuilder, + forward: ForwardFunction, +) -> tuple[Any, float, Any, int]: + import torch + + was_training = model.training + model.eval() + logits_parts: list[Any] = [] + label_parts: list[Any] = [] + with torch.no_grad(): + for row_indices in rows: + batch = build_batch(row_indices) + with context.autocast(): + logits = forward(model, batch) + logits_parts.append(logits.detach()) + label_parts.append(batch.labels.detach()) + + logits = torch.cat(logits_parts, dim=0) + labels = torch.cat(label_parts, dim=0) + with context.autocast(): + loss = torch.nn.functional.cross_entropy(logits, labels) + predictions = torch.argmax(logits, dim=1) + correct = int((predictions == labels).sum().detach().cpu().item()) + model.train(was_training) + return logits.clone(), float(loss.detach().cpu().item()), predictions.clone(), correct + + +def _metric_tensors( + *, + loss: float, + correct_count: int, + sample_count: int, + device: Any, +) -> dict[str, Any]: + import torch + + return { + "loss": torch.tensor(loss, dtype=torch.float64, device=device), + "accuracy": torch.tensor( + correct_count / sample_count, + dtype=torch.float64, + device=device, + ), + "correct_count": torch.tensor(correct_count, dtype=torch.int64, device=device), + "sample_count": torch.tensor(sample_count, dtype=torch.int64, device=device), + } + + +def _optimizer_snapshot(optimizer: Any, model: Any, scaler: Any | None) -> dict[str, Any]: + import torch + + output: dict[str, Any] = { + "optimizer": flatten_optimizer_state(optimizer, model), + } + if scaler is not None: + reference = next(model.parameters()) + state = scaler.state_dict() + output["grad_scaler"] = { + "scale": torch.tensor( + float(scaler.get_scale()), + dtype=torch.float64, + device=reference.device, + ), + "growth_factor": torch.tensor( + float(state.get("growth_factor", 1.0)), + dtype=torch.float64, + device=reference.device, + ), + "backoff_factor": torch.tensor( + float(state.get("backoff_factor", 1.0)), + dtype=torch.float64, + device=reference.device, + ), + "growth_interval": torch.tensor( + int(state.get("growth_interval", 0)), + dtype=torch.int64, + device=reference.device, + ), + "growth_tracker": torch.tensor( + int(state.get("_growth_tracker", 0)), + dtype=torch.int64, + device=reference.device, + ), + } + return output + + +def run_training_workload( + context: CaseContext, + recorder: ObservationRecorder, + *, + workload_name: str, + model: Any, + training_sample_count: int, + evaluation_sample_count: int, + training_source_indices: np.ndarray, + build_training_batch: BatchBuilder, + build_evaluation_batch: BatchBuilder, + forward: ForwardFunction, +) -> None: + """Run one fixed step-limited workload and retain its diagnostic outputs.""" + + import torch + + settings = WORKLOADS[workload_name] + training_steps = int(settings["training_steps"]) + checkpoint_steps = tuple(int(value) for value in settings["checkpoint_steps"]) + early_steps = set(int(value) for value in WORKLOAD_CAPTURE["early_parameter_state_steps"]) + optimizer = _build_optimizer(settings, model) + scaler = make_grad_scaler(context) if context.profile_id == "amp_fp16" else None + + batch_rows = _batch_schedule( + sample_count=training_sample_count, + batch_size=int(settings["batch_size"]), + step_count=training_steps, + seed=context.seed_for("training_order"), + shuffle=bool(settings["shuffle_training_data"]), + drop_last=bool(DATALOADER["drop_last"]), + ) + evaluation_rows = _evaluation_rows( + evaluation_sample_count, + int(settings["evaluation_batch_size"]), + ) + + initial_logits, initial_loss, initial_predictions, initial_correct = _evaluate( + context, + model, + rows=evaluation_rows, + build_batch=build_evaluation_batch, + forward=forward, + ) + + checkpoint_logits: dict[str, Any] = {"step_0": initial_logits} + checkpoint_metrics: dict[str, Any] = { + "step_0": _metric_tensors( + loss=initial_loss, + correct_count=initial_correct, + sample_count=evaluation_sample_count, + device=initial_logits.device, + ) + } + early_parameter_states: dict[str, Any] = {} + if 0 in early_steps: + early_parameter_states["step_0"] = clone_named_parameters(model) + + optimizer_states: dict[str, Any] = {} + if OUTPUT_CAPTURE["store_optimizer_state"]: + optimizer_states["step_0"] = _optimizer_snapshot(optimizer, model, scaler) + + training_losses: list[float] = [] + training_batch_indices: dict[str, Any] = {} + first_gradients: dict[str, Any] | None = None + + model.train() + for step_index, row_indices in enumerate(batch_rows, start=1): + batch = build_training_batch(row_indices) + source_rows = np.asarray(training_source_indices[row_indices], dtype=np.int64) + training_batch_indices[f"step_{step_index}"] = torch.from_numpy( + np.ascontiguousarray(source_rows) + ) + + optimizer.zero_grad(set_to_none=True) + with context.autocast(): + logits = forward(model, batch) + loss = torch.nn.functional.cross_entropy(logits, batch.labels) + + if scaler is None: + loss.backward() + else: + scaler.scale(loss).backward() + scaler.unscale_(optimizer) + + if step_index == 1: + first_gradients = clone_named_gradients(model) + + if scaler is None: + optimizer.step() + else: + scaler.step(optimizer) + scaler.update() + + training_losses.append(float(loss.detach().cpu().item())) + + if step_index in early_steps: + early_parameter_states[f"step_{step_index}"] = clone_named_parameters(model) + + if step_index in checkpoint_steps: + logits_at_step, loss_at_step, _, correct_at_step = _evaluate( + context, + model, + rows=evaluation_rows, + build_batch=build_evaluation_batch, + forward=forward, + ) + checkpoint_logits[f"step_{step_index}"] = logits_at_step + checkpoint_metrics[f"step_{step_index}"] = _metric_tensors( + loss=loss_at_step, + correct_count=correct_at_step, + sample_count=evaluation_sample_count, + device=logits_at_step.device, + ) + if OUTPUT_CAPTURE["store_optimizer_state"]: + optimizer_states[f"step_{step_index}"] = _optimizer_snapshot( + optimizer, + model, + scaler, + ) + + if first_gradients is None: + raise RuntimeError("The workload did not execute its first backward pass") + missing_checkpoints = { + f"step_{step}" for step in checkpoint_steps + } - set(checkpoint_logits) + if missing_checkpoints: + raise RuntimeError(f"Workload did not produce checkpoints: {sorted(missing_checkpoints)}") + + final_logits = checkpoint_logits[f"step_{training_steps}"] + final_predictions = torch.argmax(final_logits, dim=1) + final_metric_tensors = checkpoint_metrics[f"step_{training_steps}"] + final_loss = float(final_metric_tensors["loss"].detach().cpu().item()) + final_correct = int(final_metric_tensors["correct_count"].detach().cpu().item()) + + recorder.record("initial_logits", initial_logits) + recorder.record("initial_loss", initial_loss) + recorder.record("training_loss", training_losses) + recorder.record("training_batch_indices", training_batch_indices) + recorder.record("checkpoint_logits", checkpoint_logits) + recorder.record("checkpoint_metrics", checkpoint_metrics) + recorder.record("first_gradients", first_gradients) + recorder.record("early_parameter_states", early_parameter_states) + if OUTPUT_CAPTURE["store_optimizer_state"]: + recorder.record("optimizer_states", optimizer_states) + if OUTPUT_CAPTURE["store_final_parameters"]: + recorder.record("final_parameters", clone_named_parameters(model)) + recorder.record("final_predictions", final_predictions) + recorder.record( + "final_metrics", + { + "dataset_id": str(settings["dataset_id"]), + "training_steps": training_steps, + "training_batch_size": int(settings["batch_size"]), + "evaluation_batch_size": int(settings["evaluation_batch_size"]), + "training_sample_count": training_sample_count, + "evaluation_sample_count": evaluation_sample_count, + "examples_seen": int(sum(len(rows) for rows in batch_rows)), + "initial_evaluation_loss": initial_loss, + "initial_correct_count": initial_correct, + "initial_accuracy": initial_correct / evaluation_sample_count, + "final_evaluation_loss": final_loss, + "final_correct_count": final_correct, + "final_accuracy": final_correct / evaluation_sample_count, + "checkpoint_steps": list(checkpoint_steps), + "initial_prediction_count": int(initial_predictions.numel()), + "final_prediction_count": int(final_predictions.numel()), + "grad_scaler_enabled": scaler is not None, + "final_grad_scale": float(scaler.get_scale()) if scaler is not None else None, + }, + ) diff --git a/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/README.md b/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/README.md new file mode 100644 index 00000000..b08892ea --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/README.md @@ -0,0 +1,14 @@ +# Level 0 quick workloads + +This is the first thing I expect people to run when checking a new build or showing the suite to someone + +It trains and evaluates four small classifiers using the fixed generated model inputs: + +- a linear classifier +- the Level 5 MLP +- the Level 5 CNN +- the Level 5 residual multi-head attention classifier + +The MLP, CNN and attention examples call the same execution functions as Level 5. This keeps the quick demonstration representative rather than maintaining a second cut-down implementation + +The ordinary detailed tensor artefacts are still saved. The suite also writes `level_0_summary.csv` at the top of the result bundle so there is a quick human-readable view of the initial and final losses, predictions, logits, gradients and parameter norms diff --git a/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/__init__.py b/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/__init__.py new file mode 100644 index 00000000..e2235f6a --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/__init__.py @@ -0,0 +1 @@ +"""Quick model-training demonstrations used as the first CI check.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/test_demo_workloads.py b/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/test_demo_workloads.py new file mode 100644 index 00000000..d7b6f74d --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_0_smoke_workloads/test_demo_workloads.py @@ -0,0 +1,140 @@ +"""Run the four small model-training demonstrations used by Level 0.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import BLOCK_TESTS, LEVEL_0_DEMOS +from cases.common import ( + as_profile_tensor, + build_demo_summary, + build_linear_classifier, + load_module_state, + load_prepared_npz, + module_to_profile, + run_composite_block, + run_registered_case, +) +from cases.level_5_composite_models.test_attention_block import ( + run_example as run_attention_example, +) +from cases.level_5_composite_models.test_cnn_block import run_example as run_cnn_example +from cases.level_5_composite_models.test_mlp_block import run_example as run_mlp_example +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def _record_summary( + context: CaseContext, + recorder: ObservationRecorder, + *, + model_type: str, + optimiser_name: str, + labels: object, + outputs: dict[str, object], +) -> None: + recorder.record( + "summary", + build_demo_summary( + context, + model_type=model_type, + optimiser_name=optimiser_name, + labels=labels, + outputs=outputs, + ), + ) + + +def _linear_classifier(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["mlp_input"]) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + model = module_to_profile(context, build_linear_classifier()) + load_module_state(context, model, "linear_initial_state.npz") + + def forward(current_model: object, retain_activations: bool) -> tuple[object, dict[str, object]]: + logits, activations = current_model(value, return_activations=True) + return logits, activations if retain_activations else {} + + outputs = run_composite_block( + context, + recorder, + model_name="linear", + model=model, + labels=labels, + forward=forward, + optimiser_settings=LEVEL_0_DEMOS["linear"], + ) + _record_summary( + context, + recorder, + model_type="linear_classifier", + optimiser_name=str(LEVEL_0_DEMOS["linear"]["optimiser"]), + labels=labels, + outputs=outputs, + ) + + +def _mlp_classifier(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + outputs = run_mlp_example(context, recorder) + _record_summary( + context, + recorder, + model_type="multilayer_perceptron", + optimiser_name=str(BLOCK_TESTS["model_optimizers"]["mlp"]), + labels=labels, + outputs=outputs, + ) + + +def _cnn_classifier(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + labels = as_profile_tensor(context, arrays["cnn_labels"], dtype=torch.int64) + outputs = run_cnn_example(context, recorder) + _record_summary( + context, + recorder, + model_type="convolutional_neural_network", + optimiser_name=str(BLOCK_TESTS["model_optimizers"]["cnn"]), + labels=labels, + outputs=outputs, + ) + + +def _attention_classifier(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + labels = as_profile_tensor(context, arrays["attention_labels"], dtype=torch.int64) + outputs = run_attention_example(context, recorder) + _record_summary( + context, + recorder, + model_type="residual_multi_head_attention", + optimiser_name=str(BLOCK_TESTS["model_optimizers"]["attention"]), + labels=labels, + outputs=outputs, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "linear_classifier": _linear_classifier, + "mlp_classifier": _mlp_classifier, + "cnn_classifier": _cnn_classifier, + "attention_classifier": _attention_classifier, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the quick demonstration selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/__init__.py b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/__init__.py new file mode 100644 index 00000000..75d44aa9 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/__init__.py @@ -0,0 +1 @@ +"""Level 1 core tensor case modules.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_elementwise_arithmetic.py b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_elementwise_arithmetic.py new file mode 100644 index 00000000..6a5dc193 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_elementwise_arithmetic.py @@ -0,0 +1,172 @@ +"""Elementwise arithmetic cases over canonical input classes.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + if name != "special_values" + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("results", values) + + +def _add(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + _record( + recorder, + { + "ordinary_reversed": values["ordinary"] + values["ordinary"].flip(0), + "near_zero_and_ordinary": values["near_zero"] + values["ordinary"], + "broadcast": values["broadcast_left"] + values["broadcast_right"], + }, + ) + + +def _subtract(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + _record( + recorder, + { + "ordinary_reversed": values["ordinary"] - values["ordinary"].flip(0), + "mixed_sign_and_ordinary": values["mixed_sign"] - values["ordinary"], + "broadcast": values["broadcast_left"] - values["broadcast_right"], + }, + ) + + +def _multiply(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + _record( + recorder, + { + "ordinary_unit_interval": values["ordinary"] * values["unit_interval"], + "near_zero_ordinary": values["near_zero"] * values["ordinary"], + "broadcast": values["broadcast_left"] * values["broadcast_right"], + }, + ) + + +def _true_divide(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + denominator = torch.clamp(values["positive"], min=0.125) + _record( + recorder, + { + "ordinary_by_positive": torch.true_divide(values["ordinary"], denominator), + "near_zero_by_positive": torch.true_divide(values["near_zero"], denominator), + "broadcast": torch.true_divide( + values["broadcast_left"], + values["broadcast_right"].abs() + 0.5, + ), + }, + ) + + +def _floor_divide(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + denominator = torch.clamp(values["positive"], min=0.5) + _record( + recorder, + { + "ordinary_by_positive": torch.floor_divide(values["ordinary"], denominator), + "mixed_sign_by_positive": torch.floor_divide( + values["mixed_sign"], + denominator, + ), + }, + ) + + +def _remainder(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + denominator = torch.clamp(values["positive"], min=0.5) + _record( + recorder, + { + "ordinary": torch.remainder(values["ordinary"], denominator), + "mixed_sign": torch.remainder(values["mixed_sign"], denominator), + }, + ) + + +def _power(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + positive = torch.clamp(values["positive"], min=1e-3, max=16.0) + bounded = torch.clamp(values["unit_interval"], min=-0.95, max=0.95) + _record( + recorder, + { + "square": torch.pow(bounded, 2), + "cube": torch.pow(bounded, 3), + "square_root": torch.pow(positive, 0.5), + }, + ) + + +def _minimum_and_maximum(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + _record( + recorder, + { + "minimum": torch.minimum(values["ordinary"], values["mixed_sign"]), + "maximum": torch.maximum(values["ordinary"], values["mixed_sign"]), + "fmin": torch.fmin(values["ordinary"], values["mixed_sign"]), + "fmax": torch.fmax(values["ordinary"], values["mixed_sign"]), + }, + ) + + +def _clamp(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + _record( + recorder, + { + "symmetric": torch.clamp(values["ordinary"], min=-1.5, max=2.0), + "lower_only": torch.clamp_min(values["mixed_sign"], -2.5), + "upper_only": torch.clamp_max(values["mixed_sign"], 3.5), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "add": _add, + "subtract": _subtract, + "multiply": _multiply, + "true_divide": _true_divide, + "floor_divide": _floor_divide, + "remainder": _remainder, + "power": _power, + "minimum_and_maximum": _minimum_and_maximum, + "clamp": _clamp, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one elementwise arithmetic case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_indexing_and_shape.py b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_indexing_and_shape.py new file mode 100644 index 00000000..02146bed --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_indexing_and_shape.py @@ -0,0 +1,176 @@ +"""Indexing, view and shape-manipulation cases.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import ( + as_profile_tensor, + describe_tensors, + load_prepared_npz, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "indexing.npz") + return { + "source": as_profile_tensor(context, arrays["source"]), + "row_indices": as_profile_tensor(context, arrays["row_indices"]), + "column_indices": as_profile_tensor(context, arrays["column_indices"]), + "gather_indices": as_profile_tensor(context, arrays["gather_indices"]), + "boolean_mask": as_profile_tensor(context, arrays["boolean_mask"]), + "scatter_values": as_profile_tensor(context, arrays["scatter_values"]), + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("structure", describe_tensors(values)) + recorder.record("values", values) + + +def _basic_slicing(context: CaseContext, recorder: ObservationRecorder) -> None: + source = _inputs(context)["source"] + _record( + recorder, + { + "middle_block": source[1:6, 2:10, 3:12], + "strided": source[::2, 1::3, ::2], + "single_plane": source[3], + }, + ) + + +def _advanced_indexing(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + source = values["source"] + rows = values["row_indices"] + columns = values["column_indices"] + _record( + recorder, + { + "selected_rows": source[rows], + "paired_rows_and_columns": source[rows, :, columns], + "selected_columns": source[:, :, columns], + }, + ) + + +def _boolean_masking(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + source = values["source"] + mask = values["boolean_mask"] + _record( + recorder, + { + "selected": source[mask], + "filled": source.masked_fill(mask, -3.0), + }, + ) + + +def _gather(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + source = values["source"] + indices = values["gather_indices"] + _record( + recorder, + { + "gather_last_dimension": torch.gather(source, dim=2, index=indices), + "take_along_last_dimension": torch.take_along_dim( + source, + indices, + dim=2, + ), + }, + ) + + +def _scatter(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + source = values["source"] + scatter_values = values["scatter_values"] + + # Keep the indices unique along the scatter dimension + # Duplicate indices would turn this into a nondeterminism test instead + base_indices = torch.tensor( + [0, 3, 6, 9, 12], + device=context.device, + dtype=torch.int64, + ) + indices = base_indices.view(1, 1, 5).expand_as(scatter_values) + scattered = torch.zeros_like(source).scatter(2, indices, scatter_values) + added = torch.zeros_like(source).scatter_add(2, indices, scatter_values) + _record( + recorder, + { + "scatter": scattered, + "scatter_add": added, + }, + ) + + +def _reshape_and_view(context: CaseContext, recorder: ObservationRecorder) -> None: + source = _inputs(context)["source"] + _record( + recorder, + { + "flatten": source.flatten(), + "reshape_2d": source.reshape(7, 11 * 13), + "view_2d": source.view(7 * 11, 13), + "unflatten": source.flatten().unflatten(0, (7, 11, 13)), + }, + ) + + +def _transpose_and_permute(context: CaseContext, recorder: ObservationRecorder) -> None: + source = _inputs(context)["source"] + _record( + recorder, + { + "transpose": source.transpose(0, 2), + "permute": source.permute(2, 0, 1), + "movedim": source.movedim((0, 2), (2, 0)), + }, + ) + + +def _concatenate_and_stack(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + source = _inputs(context)["source"] + first = source[:3] + second = source[3:6] + _record( + recorder, + { + "concatenate": torch.cat((first, second), dim=0), + "stack": torch.stack((source[0], source[1], source[2]), dim=0), + "column_concatenate": torch.cat((source[:, :, :5], source[:, :, 5:]), dim=2), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "basic_slicing": _basic_slicing, + "advanced_indexing": _advanced_indexing, + "boolean_masking": _boolean_masking, + "gather": _gather, + "scatter": _scatter, + "reshape_and_view": _reshape_and_view, + "transpose_and_permute": _transpose_and_permute, + "concatenate_and_stack": _concatenate_and_stack, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one indexing or shape case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_tensor_creation_and_dtypes.py b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_tensor_creation_and_dtypes.py new file mode 100644 index 00000000..06142047 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_tensor_creation_and_dtypes.py @@ -0,0 +1,153 @@ +"""Core tensor creation, conversion and layout cases.""" + +from __future__ import annotations + +from collections.abc import Callable + +import numpy as np + +from cases.common import ( + as_profile_tensor, + describe_tensors, + load_prepared_npz, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _record( + recorder: ObservationRecorder, + values: dict[str, object], + *, + extra_structure: dict[str, object] | None = None, +) -> None: + structure = describe_tensors(values) + if extra_structure: + structure.update(extra_structure) + recorder.record("structure", structure) + recorder.record("values", values) + + +def _from_numpy(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + inputs = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + source = np.ascontiguousarray(inputs["ordinary"]) + cpu_view = torch.from_numpy(source) + converted = cpu_view.to(device=context.device, dtype=context.torch_dtype()) + values = { + "converted": converted, + "source_round_trip": converted.to(device="cpu"), + } + _record( + recorder, + values, + extra_structure={ + "numpy_source": { + "shape": list(source.shape), + "dtype": source.dtype.name, + "strides_bytes": list(source.strides), + "is_c_contiguous": bool(source.flags.c_contiguous), + } + }, + ) + + +def _zeros_ones_full(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + dtype = context.torch_dtype() + values = { + "zeros": torch.zeros((7, 11, 13), device=context.device, dtype=dtype), + "ones": torch.ones((7, 11, 13), device=context.device, dtype=dtype), + "full_positive": torch.full( + (7, 11, 13), + 1.25, + device=context.device, + dtype=dtype, + ), + "full_negative": torch.full( + (7, 11, 13), + -2.5, + device=context.device, + dtype=dtype, + ), + } + _record(recorder, values) + + +def _scalar_construction(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + dtype = context.torch_dtype() + values = { + "positive": torch.tensor(3.25, device=context.device, dtype=dtype), + "negative": torch.tensor(-7.5, device=context.device, dtype=dtype), + "zero": torch.tensor(0.0, device=context.device, dtype=dtype), + "integer": torch.tensor(17, device=context.device, dtype=torch.int64), + "boolean": torch.tensor(True, device=context.device, dtype=torch.bool), + } + _record(recorder, values) + + +def _dtype_conversion(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + inputs = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + source = as_profile_tensor(context, inputs["ordinary"], dtype=torch.float64) + values = { + "profile_dtype": source.to(dtype=context.torch_dtype()), + "float32": source.to(dtype=torch.float32), + "int32": source.to(dtype=torch.int32), + "boolean": source.to(dtype=torch.bool), + } + _record(recorder, values) + + +def _device_round_trip(context: CaseContext, recorder: ObservationRecorder) -> None: + inputs = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + original = as_profile_tensor(context, inputs["mixed_sign"]) + cpu_copy = original.to(device="cpu") + round_trip = cpu_copy.to(device=context.device) + values = { + "original": original, + "cpu_copy": cpu_copy, + "round_trip": round_trip, + } + _record(recorder, values) + + +def _contiguous_and_non_contiguous( + context: CaseContext, + recorder: ObservationRecorder, +) -> None: + inputs = load_prepared_npz(context, DATASET_ID, "indexing.npz") + source = as_profile_tensor(context, inputs["source"]) + transposed = source.transpose(0, 2) + narrowed = source[:, ::2, :] + values = { + "source": source, + "transposed_view": transposed, + "transposed_contiguous": transposed.contiguous(), + "strided_view": narrowed, + "strided_contiguous": narrowed.contiguous(), + } + _record(recorder, values) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "from_numpy": _from_numpy, + "zeros_ones_full": _zeros_ones_full, + "scalar_construction": _scalar_construction, + "dtype_conversion": _dtype_conversion, + "device_round_trip": _device_round_trip, + "contiguous_and_non_contiguous": _contiguous_and_non_contiguous, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one tensor creation case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_transcendental_functions.py b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_transcendental_functions.py new file mode 100644 index 00000000..b9cd9cbc --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_transcendental_functions.py @@ -0,0 +1,128 @@ +"""Transcendental and activation-function cases.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + if name in {"ordinary", "near_zero", "positive", "unit_interval", "mixed_sign"} + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("results", values) + + +def _exp_and_log(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + signed = torch.clamp(values["ordinary"], min=-8.0, max=8.0) + positive = torch.clamp(values["positive"], min=1e-4, max=20.0) + unit = torch.clamp(values["unit_interval"], min=-0.95, max=0.95) + _record( + recorder, + { + "exp": torch.exp(signed), + "expm1": torch.expm1(signed), + "log": torch.log(positive), + "log2": torch.log2(positive), + "log10": torch.log10(positive), + "log1p": torch.log1p(unit), + }, + ) + + +def _sqrt_and_rsqrt(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + positive = torch.clamp(values["positive"], min=1e-4, max=20.0) + small = torch.clamp(values["near_zero"].abs(), min=1e-4) + _record( + recorder, + { + "sqrt_positive": torch.sqrt(positive), + "rsqrt_positive": torch.rsqrt(positive), + "sqrt_small": torch.sqrt(small), + "rsqrt_small": torch.rsqrt(small), + }, + ) + + +def _trigonometric(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + angles = values["unit_interval"] * 1.25 + _record( + recorder, + { + "sin": torch.sin(angles), + "cos": torch.cos(angles), + "tan": torch.tan(angles), + "asin": torch.asin(values["unit_interval"]), + "acos": torch.acos(values["unit_interval"]), + "atan": torch.atan(values["ordinary"]), + }, + ) + + +def _hyperbolic(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + bounded = torch.clamp(values["mixed_sign"], min=-4.0, max=4.0) + inverse_input = torch.clamp(values["unit_interval"], min=-0.95, max=0.95) + _record( + recorder, + { + "sinh": torch.sinh(bounded), + "cosh": torch.cosh(bounded), + "tanh": torch.tanh(bounded), + "asinh": torch.asinh(bounded), + "atanh": torch.atanh(inverse_input), + }, + ) + + +def _sigmoid_family(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + import torch.nn.functional as functional + + values = _inputs(context) + bounded = torch.clamp(values["ordinary"], min=-12.0, max=12.0) + _record( + recorder, + { + "sigmoid": torch.sigmoid(bounded), + "log_sigmoid": functional.logsigmoid(bounded), + "softplus": functional.softplus(bounded), + "silu": functional.silu(bounded), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "exp_and_log": _exp_and_log, + "sqrt_and_rsqrt": _sqrt_and_rsqrt, + "trigonometric": _trigonometric, + "hyperbolic": _hyperbolic, + "sigmoid_family": _sigmoid_family, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one transcendental-function case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_type_promotion.py b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_type_promotion.py new file mode 100644 index 00000000..9421d17e --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_1_core_tensor/test_type_promotion.py @@ -0,0 +1,138 @@ +"""Type-promotion cases with explicit operand dtypes.""" + +from __future__ import annotations + +from collections.abc import Callable + +import numpy as np + +from cases.common import describe_tensors, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + import torch + + structure = describe_tensors(values) + structure["result_types"] = { + name: str(value.dtype).removeprefix("torch.") + for name, value in values.items() + if isinstance(value, torch.Tensor) + } + recorder.record("structure", structure) + recorder.record("values", values) + + +def _base_values(context: CaseContext) -> np.ndarray: + return np.linspace(-3.0, 3.0, num=17, dtype=np.float64) + + +def _integer_and_float(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + integer = torch.arange(-8, 9, device=context.device, dtype=torch.int32) + floating = torch.tensor( + _base_values(context), + device=context.device, + dtype=context.torch_dtype(), + ) + _record( + recorder, + { + "add": integer + floating, + "multiply": integer * floating, + "true_divide": integer / (floating.abs() + 0.5), + }, + ) + + +def _float_widths(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + base = _base_values(context) + float16 = torch.tensor(base, device=context.device, dtype=torch.float16) + float32 = torch.tensor(base, device=context.device, dtype=torch.float32) + float64 = torch.tensor(base, device=context.device, dtype=torch.float64) + profile = torch.tensor(base, device=context.device, dtype=context.torch_dtype()) + lower = float32 if context.torch_dtype() == torch.float64 else float16 + _record( + recorder, + { + "float16_plus_float32": float16 + float32, + "float32_plus_float64": float32 + float64, + "lower_plus_profile": lower + profile, + }, + ) + + +def _scalar_and_tensor(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + tensor = torch.tensor( + _base_values(context), + device=context.device, + dtype=context.torch_dtype(), + ) + _record( + recorder, + { + "python_integer": tensor + 3, + "python_float": tensor + 0.25, + "zero_dimensional_integer": tensor + + torch.tensor(3, device=context.device, dtype=torch.int64), + "zero_dimensional_float": tensor + + torch.tensor(0.25, device=context.device, dtype=torch.float32), + }, + ) + + +def _boolean_and_numeric(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + numeric = torch.tensor( + _base_values(context), + device=context.device, + dtype=context.torch_dtype(), + ) + boolean = numeric > 0 + _record( + recorder, + { + "add": boolean + numeric, + "multiply": boolean * numeric, + "where": torch.where(boolean, numeric, -numeric), + }, + ) + + +def _complex_and_real(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + real_dtype = context.torch_dtype() + complex_dtype = torch.complex128 if real_dtype == torch.float64 else torch.complex64 + real = torch.tensor(_base_values(context), device=context.device, dtype=real_dtype) + imaginary = torch.linspace(1.0, 2.0, 17, device=context.device, dtype=real_dtype) + complex_values = torch.complex(real, imaginary).to(dtype=complex_dtype) + _record( + recorder, + { + "add": complex_values + real, + "multiply": complex_values * real, + "divide": complex_values / (real.abs() + 0.5), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "integer_and_float": _integer_and_float, + "float_widths": _float_widths, + "scalar_and_tensor": _scalar_and_tensor, + "boolean_and_numeric": _boolean_and_numeric, + "complex_and_real": _complex_and_real, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one type-promotion case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/__init__.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/__init__.py new file mode 100644 index 00000000..2860cb6b --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/__init__.py @@ -0,0 +1 @@ +"""Level 2 numerical kernel cases.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_convolution.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_convolution.py new file mode 100644 index 00000000..5f1ed250 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_convolution.py @@ -0,0 +1,118 @@ +"""Convolution cases using fixed inputs, weights and biases.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "convolutions.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("results", values) + + +def _conv1d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + values = _inputs(context) + source = values["conv1d_input"] + weight = values["conv1d_weight"] + bias = values["conv1d_bias"] + _record( + recorder, + { + "valid": functional.conv1d(source, weight, bias), + "same_length": functional.conv1d(source, weight, bias, padding=2), + "strided": functional.conv1d(source, weight, bias, stride=2, padding=2), + "dilated": functional.conv1d(source, weight, bias, dilation=2, padding=4), + }, + ) + + +def _conv2d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + values = _inputs(context) + source = values["conv2d_input"] + weight = values["conv2d_weight"] + bias = values["conv2d_bias"] + _record( + recorder, + { + "valid": functional.conv2d(source, weight, bias), + "same_shape": functional.conv2d(source, weight, bias, padding=1), + "strided": functional.conv2d(source, weight, bias, stride=2, padding=1), + "dilated": functional.conv2d(source, weight, bias, dilation=2, padding=2), + }, + ) + + +def _grouped_conv2d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + values = _inputs(context) + source = values["grouped_conv2d_input"] + weight = values["grouped_conv2d_weight"] + bias = values["grouped_conv2d_bias"] + _record( + recorder, + { + "groups_two": functional.conv2d( + source, + weight, + bias, + padding=1, + groups=2, + ), + "groups_two_strided": functional.conv2d( + source, + weight, + bias, + stride=2, + padding=1, + groups=2, + ), + }, + ) + + +def _conv3d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + values = _inputs(context) + source = values["conv3d_input"] + weight = values["conv3d_weight"] + bias = values["conv3d_bias"] + _record( + recorder, + { + "valid": functional.conv3d(source, weight, bias), + "same_shape": functional.conv3d(source, weight, bias, padding=1), + "strided": functional.conv3d(source, weight, bias, stride=2, padding=1), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "conv1d": _conv1d, + "conv2d": _conv2d, + "grouped_conv2d": _grouped_conv2d, + "conv3d": _conv3d, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one convolution case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_eigensystems.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_eigensystems.py new file mode 100644 index 00000000..ac303de2 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_eigensystems.py @@ -0,0 +1,67 @@ +"""Symmetric eigensystem cases with residual and subspace outputs.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "linear_algebra.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _eigensystem_invariants(matrix: object, *, degenerate_count: int) -> dict[str, object]: + import torch + + eigenvalues, eigenvectors = torch.linalg.eigh(matrix) + reconstructed_action = matrix @ eigenvectors + scaled_vectors = eigenvectors * eigenvalues.unsqueeze(0) + identity = torch.eye( + eigenvectors.shape[1], + dtype=eigenvectors.dtype, + device=eigenvectors.device, + ) + subspace = eigenvectors[:, :degenerate_count] + return { + "eigenvalues": eigenvalues, + "eigenvectors": eigenvectors, + "eigen_residual": reconstructed_action - scaled_vectors, + "orthogonality_residual": eigenvectors.transpose(-2, -1) @ eigenvectors - identity, + "leading_subspace_projector": subspace @ subspace.transpose(-2, -1), + } + + +def _symmetric_distinct(context: CaseContext, recorder: ObservationRecorder) -> None: + matrix = _inputs(context)["well_conditioned_matrix"] + recorder.record( + "invariants", + _eigensystem_invariants(matrix, degenerate_count=1), + ) + + +def _symmetric_degenerate(context: CaseContext, recorder: ObservationRecorder) -> None: + matrix = _inputs(context)["degenerate_symmetric_matrix"] + recorder.record( + "invariants", + _eigensystem_invariants(matrix, degenerate_count=3), + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "symmetric_distinct": _symmetric_distinct, + "symmetric_degenerate": _symmetric_degenerate, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one eigensystem case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_factorisations.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_factorisations.py new file mode 100644 index 00000000..71e1ccd2 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_factorisations.py @@ -0,0 +1,93 @@ +"""Matrix factorisation cases with reconstruction and orthogonality checks.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "linear_algebra.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("invariants", values) + + +def _qr(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + matrix = _inputs(context)["rectangular_matrix"] + q, r = torch.linalg.qr(matrix, mode="reduced") + identity = torch.eye(q.shape[1], dtype=q.dtype, device=q.device) + reconstruction = q @ r + _record( + recorder, + { + "q": q, + "r": r, + "reconstruction": reconstruction, + "reconstruction_residual": reconstruction - matrix, + "orthogonality_residual": q.transpose(-2, -1) @ q - identity, + }, + ) + + +def _svd(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + matrix = _inputs(context)["svd_matrix"] + u, singular_values, vh = torch.linalg.svd(matrix, full_matrices=False) + reconstruction = (u * singular_values.unsqueeze(0)) @ vh + u_identity = torch.eye(u.shape[1], dtype=u.dtype, device=u.device) + v_identity = torch.eye(vh.shape[0], dtype=vh.dtype, device=vh.device) + _record( + recorder, + { + "u": u, + "singular_values": singular_values, + "vh": vh, + "reconstruction": reconstruction, + "reconstruction_residual": reconstruction - matrix, + "u_orthogonality_residual": u.transpose(-2, -1) @ u - u_identity, + "v_orthogonality_residual": vh @ vh.transpose(-2, -1) - v_identity, + }, + ) + + +def _cholesky(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + matrix = _inputs(context)["positive_definite_matrix"] + factor = torch.linalg.cholesky(matrix) + reconstruction = factor @ factor.transpose(-2, -1) + _record( + recorder, + { + "factor": factor, + "reconstruction": reconstruction, + "reconstruction_residual": reconstruction - matrix, + "strict_upper_triangle": torch.triu(factor, diagonal=1), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "qr": _qr, + "svd": _svd, + "cholesky": _cholesky, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one factorisation case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_fft.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_fft.py new file mode 100644 index 00000000..84013e8a --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_fft.py @@ -0,0 +1,103 @@ +"""FFT cases which retain both transforms and inverse reconstructions.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "fft.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record( + recorder: ObservationRecorder, + *, + transforms: dict[str, object], + reconstructions: dict[str, object], +) -> None: + recorder.record("transforms", transforms) + recorder.record("reconstructions", reconstructions) + + +def _fft_1d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + source = _inputs(context)["complex_1d"] + transform = torch.fft.fft(source) + orthonormal_transform = torch.fft.fft(source, norm="ortho") + _record( + recorder, + transforms={"default": transform, "orthonormal": orthonormal_transform}, + reconstructions={ + "default": torch.fft.ifft(transform), + "orthonormal": torch.fft.ifft(orthonormal_transform, norm="ortho"), + }, + ) + + +def _fft_2d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + source = _inputs(context)["complex_2d"] + transform = torch.fft.fft2(source) + shifted = torch.fft.fftshift(transform) + _record( + recorder, + transforms={"default": transform, "shifted": shifted}, + reconstructions={ + "default": torch.fft.ifft2(transform), + "shifted": torch.fft.ifft2(torch.fft.ifftshift(shifted)), + }, + ) + + +def _real_fft(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + source_1d = _inputs(context)["real_1d"] + source_2d = _inputs(context)["real_2d"] + transform_1d = torch.fft.rfft(source_1d) + transform_2d = torch.fft.rfft2(source_2d) + _record( + recorder, + transforms={"one_dimensional": transform_1d, "two_dimensional": transform_2d}, + reconstructions={ + "one_dimensional": torch.fft.irfft(transform_1d, n=source_1d.shape[0]), + "two_dimensional": torch.fft.irfft2(transform_2d, s=source_2d.shape), + }, + ) + + +def _inverse_round_trip(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + source = _inputs(context)["complex_2d"] + inverse_transform = torch.fft.ifftn(source) + _record( + recorder, + transforms={"inverse": inverse_transform}, + reconstructions={"forward_after_inverse": torch.fft.fftn(inverse_transform)}, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "fft_1d": _fft_1d, + "fft_2d": _fft_2d, + "real_fft": _real_fft, + "inverse_round_trip": _inverse_round_trip, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one FFT case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_linear_solve.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_linear_solve.py new file mode 100644 index 00000000..6a4c11f2 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_linear_solve.py @@ -0,0 +1,107 @@ +"""Linear solve cases with residuals against the original equations.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "linear_algebra.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record( + recorder: ObservationRecorder, + *, + solutions: dict[str, object], + residuals: dict[str, object], +) -> None: + recorder.record("solutions", solutions) + recorder.record("residuals", residuals) + + +def _well_conditioned_solve( + context: CaseContext, + recorder: ObservationRecorder, +) -> None: + import torch + + values = _inputs(context) + matrix = values["well_conditioned_matrix"] + right_hand_side = values["well_conditioned_rhs"] + solution = torch.linalg.solve(matrix, right_hand_side) + _record( + recorder, + solutions={"solution": solution}, + residuals={"equation": matrix @ solution - right_hand_side}, + ) + + +def _ill_conditioned_solve( + context: CaseContext, + recorder: ObservationRecorder, +) -> None: + import torch + + values = _inputs(context) + matrix = values["ill_conditioned_matrix"] + right_hand_side = values["ill_conditioned_rhs"] + solution = torch.linalg.solve(matrix, right_hand_side) + _record( + recorder, + solutions={"solution": solution}, + residuals={"equation": matrix @ solution - right_hand_side}, + ) + + +def _matrix_inverse(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + matrix = _inputs(context)["well_conditioned_matrix"] + inverse = torch.linalg.inv(matrix) + identity = torch.eye(matrix.shape[0], dtype=matrix.dtype, device=matrix.device) + _record( + recorder, + solutions={"inverse": inverse}, + residuals={ + "left_identity": matrix @ inverse - identity, + "right_identity": inverse @ matrix - identity, + }, + ) + + +def _cholesky_solve(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + matrix = values["positive_definite_matrix"] + right_hand_side = values["well_conditioned_rhs"] + factor = torch.linalg.cholesky(matrix) + solution = torch.cholesky_solve(right_hand_side, factor) + _record( + recorder, + solutions={"factor": factor, "solution": solution}, + residuals={"equation": matrix @ solution - right_hand_side}, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "well_conditioned_solve": _well_conditioned_solve, + "ill_conditioned_solve": _ill_conditioned_solve, + "matrix_inverse": _matrix_inverse, + "cholesky_solve": _cholesky_solve, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one linear solve case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_matrix_multiplication.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_matrix_multiplication.py new file mode 100644 index 00000000..7d80218e --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_matrix_multiplication.py @@ -0,0 +1,121 @@ +"""Matrix multiplication cases over fixed irregular dimensions.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "matrix_operations.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("results", values) + + +def _matrix_vector(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + left = values["left"] + vector = values["vector"] + _record( + recorder, + { + "mv": torch.mv(left, vector), + "matmul": torch.matmul(left, vector), + "transposed_mv": torch.mv(left.transpose(0, 1), left[:, 0]), + }, + ) + + +def _matrix_matrix(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + left = values["left"] + right = values["right"] + _record( + recorder, + { + "mm": torch.mm(left, right), + "matmul": torch.matmul(left, right), + "left_gram": left.transpose(0, 1) @ left, + "right_gram": right @ right.transpose(0, 1), + }, + ) + + +def _batched_matrix_matrix(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + batch_left = values["batch_left"] + batch_right = values["batch_right"] + _record( + recorder, + { + "bmm": torch.bmm(batch_left, batch_right), + "matmul": torch.matmul(batch_left, batch_right), + "broadcast_right": torch.matmul(batch_left, batch_right[0]), + }, + ) + + +def _einsum(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + left = values["einsum_left"] + right = values["einsum_right"] + _record( + recorder, + { + "contract_last_dimension": torch.einsum("bij,jk->bik", left, right), + "batch_gram": torch.einsum("bij,bik->bjk", left, left), + "diagonal_trace": torch.einsum("bii->b", left[:, :, :7]), + }, + ) + + +def _inner_and_outer(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + vector = values["vector"] + second = values["left"][0] + short_left = vector[:31] + short_right = second[:31] + _record( + recorder, + { + "inner": torch.inner(vector, second), + "dot": torch.dot(vector, second), + "outer": torch.outer(short_left, short_right), + "ger": torch.ger(short_left, short_right), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "matrix_vector": _matrix_vector, + "matrix_matrix": _matrix_matrix, + "batched_matrix_matrix": _batched_matrix_matrix, + "einsum": _einsum, + "inner_and_outer": _inner_and_outer, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one matrix operation case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_pooling.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_pooling.py new file mode 100644 index 00000000..cdd3d7e6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_pooling.py @@ -0,0 +1,154 @@ +"""Pooling cases over the prepared convolution inputs.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "convolutions.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + if name.endswith("_input") + } + + +def _empty_indices(context: CaseContext) -> object: + import torch + + return torch.empty(0, dtype=torch.int64, device=context.device) + + +def _record( + recorder: ObservationRecorder, + *, + values: dict[str, object], + indices: dict[str, object], +) -> None: + recorder.record("values", values) + recorder.record("indices", indices) + + +def _max_pool1d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + source = _inputs(context)["conv1d_input"] + values, indices = functional.max_pool1d( + source, + kernel_size=3, + stride=2, + padding=1, + return_indices=True, + ) + ceil_values, ceil_indices = functional.max_pool1d( + source, + kernel_size=4, + stride=3, + padding=1, + ceil_mode=True, + return_indices=True, + ) + _record( + recorder, + values={"standard": values, "ceil_mode": ceil_values}, + indices={"standard": indices, "ceil_mode": ceil_indices}, + ) + + +def _max_pool2d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + source = _inputs(context)["conv2d_input"] + values, indices = functional.max_pool2d( + source, + kernel_size=(3, 2), + stride=(2, 2), + padding=(1, 0), + return_indices=True, + ) + dilated_values, dilated_indices = functional.max_pool2d( + source, + kernel_size=3, + stride=2, + padding=1, + dilation=2, + return_indices=True, + ) + _record( + recorder, + values={"standard": values, "dilated": dilated_values}, + indices={"standard": indices, "dilated": dilated_indices}, + ) + + +def _average_pool2d(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + source = _inputs(context)["conv2d_input"] + _record( + recorder, + values={ + "include_padding": functional.avg_pool2d( + source, + kernel_size=3, + stride=2, + padding=1, + count_include_pad=True, + ), + "exclude_padding": functional.avg_pool2d( + source, + kernel_size=3, + stride=2, + padding=1, + count_include_pad=False, + ), + "divisor_override": functional.avg_pool2d( + source, + kernel_size=2, + stride=2, + divisor_override=5, + ), + }, + indices={"not_applicable": _empty_indices(context)}, + ) + + +def _adaptive_average_pool2d( + context: CaseContext, + recorder: ObservationRecorder, +) -> None: + import torch.nn.functional as functional + + source = _inputs(context)["conv2d_input"] + _record( + recorder, + values={ + "one_by_one": functional.adaptive_avg_pool2d(source, output_size=(1, 1)), + "irregular": functional.adaptive_avg_pool2d(source, output_size=(5, 7)), + "partially_preserved": functional.adaptive_avg_pool2d( + source, + output_size=(None, 4), + ), + }, + indices={"not_applicable": _empty_indices(context)}, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "max_pool1d": _max_pool1d, + "max_pool2d": _max_pool2d, + "average_pool2d": _average_pool2d, + "adaptive_average_pool2d": _adaptive_average_pool2d, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one pooling case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_reductions_and_statistics.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_reductions_and_statistics.py new file mode 100644 index 00000000..610f8be0 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_reductions_and_statistics.py @@ -0,0 +1,171 @@ +"""Reduction and statistics cases over canonical prepared inputs.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "reductions.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("results", values) + + +def _sum_and_mean(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + positive = values["positive"] + mixed_sign = values["mixed_sign"] + cube = values["cube"] + _record( + recorder, + { + "positive_sum_all": positive.sum(), + "positive_sum_rows": positive.sum(dim=1), + "positive_mean_columns": positive.mean(dim=0), + "mixed_sign_sum_all": mixed_sign.sum(), + "mixed_sign_mean_rows": mixed_sign.mean(dim=1), + "cube_sum_last_dimension": cube.sum(dim=-1), + "cube_mean_first_two_dimensions": cube.mean(dim=(0, 1)), + }, + ) + + +def _variance_and_standard_deviation( + context: CaseContext, + recorder: ObservationRecorder, +) -> None: + values = _inputs(context) + mixed_sign = values["mixed_sign"] + cube = values["cube"] + _record( + recorder, + { + "variance_population_all": mixed_sign.var(correction=0), + "variance_sample_rows": mixed_sign.var(dim=1, correction=1), + "standard_deviation_population_columns": mixed_sign.std( + dim=0, + correction=0, + ), + "cube_variance_last_dimension": cube.var(dim=-1, correction=0), + "cube_standard_deviation_first_dimension": cube.std( + dim=0, + correction=1, + ), + }, + ) + + +def _minimum_and_maximum(context: CaseContext, recorder: ObservationRecorder) -> None: + values = _inputs(context) + mixed_sign = values["mixed_sign"] + cube = values["cube"] + row_minimum = mixed_sign.min(dim=1) + column_maximum = mixed_sign.max(dim=0) + cube_minimum = cube.amin(dim=(1, 2)) + cube_maximum = cube.amax(dim=(0, 2)) + _record( + recorder, + { + "global_minimum": mixed_sign.min(), + "global_maximum": mixed_sign.max(), + "row_minimum_values": row_minimum.values, + "row_minimum_indices": row_minimum.indices, + "column_maximum_values": column_maximum.values, + "column_maximum_indices": column_maximum.indices, + "cube_minimum": cube_minimum, + "cube_maximum": cube_maximum, + }, + ) + + +def _cumulative_operations(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + mixed_sign = values["mixed_sign"] + positive = values["positive"] + integer_values = values["integer_values"] + + # Keep cumprod close to one so lower precision profiles do not overflow immediately + stable_product_values = 1.0 + (positive[:7, :17] - 5.0) * 1e-3 + _record( + recorder, + { + "mixed_sign_cumsum_rows": torch.cumsum(mixed_sign, dim=1), + "mixed_sign_cumsum_columns": torch.cumsum(mixed_sign, dim=0), + "stable_cumprod_rows": torch.cumprod(stable_product_values, dim=1), + "integer_cumsum_rows": torch.cumsum(integer_values, dim=1), + "mixed_sign_logcumsumexp_rows": torch.logcumsumexp(mixed_sign, dim=1), + }, + ) + + +def _vector_and_matrix_norms(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + mixed_sign = values["mixed_sign"] + cube = values["cube"] + vector = mixed_sign[0] + matrix = mixed_sign[:31, :29] + _record( + recorder, + { + "vector_l1": torch.linalg.vector_norm(vector, ord=1), + "vector_l2": torch.linalg.vector_norm(vector, ord=2), + "vector_infinity": torch.linalg.vector_norm(vector, ord=float("inf")), + "matrix_frobenius": torch.linalg.matrix_norm(matrix, ord="fro"), + "matrix_one_norm": torch.linalg.matrix_norm(matrix, ord=1), + "batched_vector_norm": torch.linalg.vector_norm(cube, dim=-1), + }, + ) + + +def _cancellation_heavy_sum(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + values = _inputs(context) + cancellation = values["cancellation"] + flattened = cancellation.reshape(-1) + ascending = torch.sort(flattened).values + descending = ascending.flip(0) + complete_pattern_length = (flattened.numel() // 5) * 5 + paired = flattened[:complete_pattern_length].reshape(-1, 5) + _record( + recorder, + { + "source_order_sum": flattened.sum(), + "ascending_order_sum": ascending.sum(), + "descending_order_sum": descending.sum(), + "row_sums": cancellation.sum(dim=1), + "pattern_sums": paired.sum(dim=1), + "mean": flattened.mean(), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "sum_and_mean": _sum_and_mean, + "variance_and_standard_deviation": _variance_and_standard_deviation, + "minimum_and_maximum": _minimum_and_maximum, + "cumulative_operations": _cumulative_operations, + "vector_and_matrix_norms": _vector_and_matrix_norms, + "cancellation_heavy_sum": _cancellation_heavy_sum, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one reduction or statistics case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_special_functions.py b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_special_functions.py new file mode 100644 index 00000000..05500762 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_2_numerical_kernels/test_special_functions.py @@ -0,0 +1,100 @@ +"""Special mathematical function cases over bounded canonical inputs.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _inputs(context: CaseContext) -> dict[str, object]: + arrays = load_prepared_npz(context, DATASET_ID, "special_functions.npz") + return { + name: as_profile_tensor(context, value) + for name, value in arrays.items() + } + + +def _record(recorder: ObservationRecorder, values: dict[str, object]) -> None: + recorder.record("results", values) + + +def _erf_family(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + signed = _inputs(context)["signed"] + inverse_input = torch.tanh(signed / 4.0) * 0.95 + _record( + recorder, + { + "erf": torch.erf(signed), + "erfc": torch.erfc(signed), + "erfinv": torch.erfinv(inverse_input), + }, + ) + + +def _gamma_family(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + gamma_inputs = _inputs(context)["gamma_inputs"] + _record( + recorder, + { + "lgamma": torch.lgamma(gamma_inputs), + "gammaln": torch.special.gammaln(gamma_inputs), + "digamma": torch.digamma(gamma_inputs), + "polygamma_one": torch.polygamma(1, gamma_inputs), + }, + ) + + +def _softmax_and_log_softmax( + context: CaseContext, + recorder: ObservationRecorder, +) -> None: + import torch + + matrix = _inputs(context)["softmax_matrix"] + _record( + recorder, + { + "softmax_rows": torch.softmax(matrix, dim=1), + "log_softmax_rows": torch.log_softmax(matrix, dim=1), + "logsumexp_rows": torch.logsumexp(matrix, dim=1), + "softmax_columns": torch.softmax(matrix, dim=0), + }, + ) + + +def _logit_and_expit(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + probabilities = _inputs(context)["probabilities"] + logits = torch.logit(probabilities) + _record( + recorder, + { + "logit": logits, + "expit": torch.special.expit(logits), + "sigmoid": torch.sigmoid(logits), + "logit_with_epsilon": torch.logit(probabilities, eps=1e-5), + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "erf_family": _erf_family, + "gamma_family": _gamma_family, + "softmax_and_log_softmax": _softmax_and_log_softmax, + "logit_and_expit": _logit_and_expit, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one special-function case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/__init__.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/__init__.py new file mode 100644 index 00000000..bec6febe --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/__init__.py @@ -0,0 +1 @@ +"""Level 3 autograd and learning-component cases.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_attention.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_attention.py new file mode 100644 index 00000000..30721396 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_attention.py @@ -0,0 +1,159 @@ +"""Attention forward and backward cases with fixed projections and masks.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import MODEL_ARCHITECTURES +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def _inputs(context: CaseContext) -> tuple[object, object, dict[str, object]]: + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + state_arrays = load_prepared_npz(context, DATASET_ID, "attention_initial_state.npz") + value = as_profile_tensor(context, arrays["attention_input"], requires_grad=True) + mask = as_profile_tensor(context, arrays["attention_padding_mask"]) + state = {name: as_profile_tensor(context, array) for name, array in state_arrays.items()} + return value, mask, state + + +def _functional_attention( + context: CaseContext, + recorder: ObservationRecorder, + *, + use_mask: bool, +) -> None: + import math + import torch + + value, padding_mask, state = _inputs(context) + architecture = MODEL_ARCHITECTURES["attention"] + head_count = architecture["heads"] + head_size = architecture["embedding_size"] // head_count + + q_weight = state["q_proj.weight"].detach().clone().requires_grad_(True) + k_weight = state["k_proj.weight"].detach().clone().requires_grad_(True) + v_weight = state["v_proj.weight"].detach().clone().requires_grad_(True) + q_bias = state["q_proj.bias"].detach().clone().requires_grad_(True) + k_bias = state["k_proj.bias"].detach().clone().requires_grad_(True) + v_bias = state["v_proj.bias"].detach().clone().requires_grad_(True) + + query = torch.nn.functional.linear(value, q_weight, q_bias) + key = torch.nn.functional.linear(value, k_weight, k_bias) + projected_value = torch.nn.functional.linear(value, v_weight, v_bias) + + def split_heads(tensor: object) -> object: + return tensor.reshape(tensor.shape[0], tensor.shape[1], head_count, head_size).transpose(1, 2) + + query_heads = split_heads(query) + key_heads = split_heads(key) + value_heads = split_heads(projected_value) + scores = query_heads @ key_heads.transpose(-2, -1) / math.sqrt(head_size) + if use_mask: + # Keep at least one key visible even if a prepared row was fully masked + padding_mask = padding_mask.clone() + padding_mask[:, 0] = False + scores = scores.masked_fill(padding_mask[:, None, None, :], float("-inf")) + weights = torch.softmax(scores, dim=-1) + attended = weights @ value_heads + output = attended.transpose(1, 2).contiguous().reshape_as(value) + loss = output.square().mean() + loss.backward() + + recorder.record( + "forward", + { + "query": query, + "key": key, + "value": projected_value, + "attention_weights": weights, + "output": output, + }, + ) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record("input_gradients", {"value": value.grad.detach().clone()}) + recorder.record( + "parameter_gradients", + { + "q_weight": q_weight.grad.detach().clone(), + "k_weight": k_weight.grad.detach().clone(), + "v_weight": v_weight.grad.detach().clone(), + "q_bias": q_bias.grad.detach().clone(), + "k_bias": k_bias.grad.detach().clone(), + "v_bias": v_bias.grad.detach().clone(), + }, + ) + + +def _scaled_dot_product(context: CaseContext, recorder: ObservationRecorder) -> None: + _functional_attention(context, recorder, use_mask=False) + + +def _masked_scaled_dot_product(context: CaseContext, recorder: ObservationRecorder) -> None: + _functional_attention(context, recorder, use_mask=True) + + +def _multihead_attention(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + value, padding_mask, state = _inputs(context) + architecture = MODEL_ARCHITECTURES["attention"] + module = torch.nn.MultiheadAttention( + architecture["embedding_size"], + architecture["heads"], + dropout=0.0, + batch_first=True, + ).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + module.in_proj_weight.copy_( + torch.cat( + [state["q_proj.weight"], state["k_proj.weight"], state["v_proj.weight"]], + dim=0, + ) + ) + module.in_proj_bias.copy_( + torch.cat( + [state["q_proj.bias"], state["k_proj.bias"], state["v_proj.bias"]], + dim=0, + ) + ) + module.out_proj.weight.copy_(state["out_proj.weight"]) + module.out_proj.bias.copy_(state["out_proj.bias"]) + padding_mask = padding_mask.clone() + padding_mask[:, 0] = False + output, weights = module( + value, + value, + value, + key_padding_mask=padding_mask, + need_weights=True, + average_attn_weights=False, + ) + loss = output.square().mean() + loss.backward() + recorder.record("forward", {"output": output, "attention_weights": weights}) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record("input_gradients", {"value": value.grad.detach().clone()}) + recorder.record( + "parameter_gradients", + { + name: parameter.grad.detach().clone() + for name, parameter in module.named_parameters() + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "scaled_dot_product": _scaled_dot_product, + "masked_scaled_dot_product": _masked_scaled_dot_product, + "multihead_attention": _multihead_attention, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one attention case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_autograd_elementwise.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_autograd_elementwise.py new file mode 100644 index 00000000..243f8176 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_autograd_elementwise.py @@ -0,0 +1,122 @@ +"""Autograd cases built from small elementwise computation graphs.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _finish( + recorder: ObservationRecorder, + *, + forward: dict[str, object], + loss: object, + inputs: dict[str, object], + parameters: dict[str, object], +) -> None: + loss.backward() + recorder.record("forward", forward) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record( + "input_gradients", + {name: value.grad.detach().clone() for name, value in inputs.items()}, + ) + recorder.record( + "parameter_gradients", + {name: value.grad.detach().clone() for name, value in parameters.items()}, + ) + + +def _scalar_chain(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + value = as_profile_tensor(context, arrays["positive"][:64], requires_grad=True) + scale = torch.tensor(1.25, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + bias = torch.tensor(-0.1, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + affine = value * scale + bias + output = torch.exp(affine).log1p() + loss = output.square().mean() + _finish( + recorder, + forward={"affine": affine, "output": output}, + loss=loss, + inputs={"value": value}, + parameters={"scale": scale, "bias": bias}, + ) + + +def _branching_graph(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + value = as_profile_tensor(context, arrays["mixed_sign"][:96], requires_grad=True) + frequency = torch.tensor(0.75, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + offset = torch.tensor(0.2, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + sine_branch = torch.sin(value * frequency) + cosine_branch = torch.cos(value + offset) + output = sine_branch * cosine_branch + sine_branch + loss = output.square().mean() + _finish( + recorder, + forward={"sine_branch": sine_branch, "cosine_branch": cosine_branch, "output": output}, + loss=loss, + inputs={"value": value}, + parameters={"frequency": frequency, "offset": offset}, + ) + + +def _reused_tensor(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "elementwise.npz") + value = as_profile_tensor(context, arrays["ordinary"][:80], requires_grad=True) + scale = torch.tensor(1.1, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + shared = torch.tanh(value * scale) + output = shared.square() + shared * shared.mean() + shared + loss = output.abs().mean() + _finish( + recorder, + forward={"shared": shared, "output": output}, + loss=loss, + inputs={"value": value}, + parameters={"scale": scale}, + ) + + +def _reduction_graph(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "reductions.npz") + value = as_profile_tensor(context, arrays["mixed_sign"][:31, :29], requires_grad=True) + scale = torch.tensor(0.9, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + offset = torch.tensor(0.15, device=context.device, dtype=context.torch_dtype(), requires_grad=True) + transformed = value * scale + offset + row_means = transformed.mean(dim=1) + output = torch.log1p(row_means.square()) + loss = output.sum() + _finish( + recorder, + forward={"transformed": transformed, "row_means": row_means, "output": output}, + loss=loss, + inputs={"value": value}, + parameters={"scale": scale, "offset": offset}, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "scalar_chain": _scalar_chain, + "branching_graph": _branching_graph, + "reused_tensor": _reused_tensor, + "reduction_graph": _reduction_graph, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one elementwise autograd case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_autograd_matrix_ops.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_autograd_matrix_ops.py new file mode 100644 index 00000000..9d111b1d --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_autograd_matrix_ops.py @@ -0,0 +1,117 @@ +"""Autograd cases for matrix, convolution and solve operations.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "numerical_inputs_v1" + + +def _finish( + recorder: ObservationRecorder, + *, + forward: dict[str, object], + loss: object, + inputs: dict[str, object], + parameters: dict[str, object], +) -> None: + loss.backward() + recorder.record("forward", forward) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record( + "input_gradients", + {name: value.grad.detach().clone() for name, value in inputs.items()}, + ) + recorder.record( + "parameter_gradients", + {name: value.grad.detach().clone() for name, value in parameters.items()}, + ) + + +def _matrix_multiplication(context: CaseContext, recorder: ObservationRecorder) -> None: + arrays = load_prepared_npz(context, DATASET_ID, "matrix_operations.npz") + left = as_profile_tensor(context, arrays["left"], requires_grad=True) + weight = as_profile_tensor(context, arrays["right"], requires_grad=True) + output = left @ weight + loss = output.square().mean() + _finish( + recorder, + forward={"output": output, "row_summary": output.mean(dim=1)}, + loss=loss, + inputs={"left": left}, + parameters={"weight": weight}, + ) + + +def _batched_matrix_multiplication(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "matrix_operations.npz") + left = as_profile_tensor(context, arrays["batch_left"], requires_grad=True) + weight = as_profile_tensor(context, arrays["batch_right"], requires_grad=True) + output = torch.bmm(left, weight) + loss = output.abs().mean() + _finish( + recorder, + forward={"output": output, "batch_summary": output.mean(dim=(1, 2))}, + loss=loss, + inputs={"left": left}, + parameters={"weight": weight}, + ) + + +def _convolution(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + arrays = load_prepared_npz(context, DATASET_ID, "convolutions.npz") + value = as_profile_tensor(context, arrays["conv2d_input"], requires_grad=True) + weight = as_profile_tensor(context, arrays["conv2d_weight"], requires_grad=True) + bias = as_profile_tensor(context, arrays["conv2d_bias"], requires_grad=True) + output = functional.conv2d(value, weight, bias, stride=2, padding=1) + loss = output.square().mean() + _finish( + recorder, + forward={"output": output, "channel_means": output.mean(dim=(0, 2, 3))}, + loss=loss, + inputs={"value": value}, + parameters={"weight": weight, "bias": bias}, + ) + + +def _linear_solve(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "linear_algebra.npz") + right_hand_side = as_profile_tensor( + context, arrays["well_conditioned_rhs"], requires_grad=True + ) + matrix = as_profile_tensor( + context, arrays["well_conditioned_matrix"], requires_grad=True + ) + solution = torch.linalg.solve(matrix, right_hand_side) + residual = matrix @ solution - right_hand_side + loss = solution.square().mean() + residual.square().mean() + _finish( + recorder, + forward={"solution": solution, "residual": residual}, + loss=loss, + inputs={"right_hand_side": right_hand_side}, + parameters={"matrix": matrix}, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "matrix_multiplication": _matrix_multiplication, + "batched_matrix_multiplication": _batched_matrix_multiplication, + "convolution": _convolution, + "linear_solve": _linear_solve, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one matrix-operation autograd case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_losses.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_losses.py new file mode 100644 index 00000000..9167f3db --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_losses.py @@ -0,0 +1,115 @@ +"""Loss-function cases covering reductions and input gradients.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def _record_reductions( + recorder: ObservationRecorder, + builders: dict[str, Callable[[], tuple[object, object]]], +) -> None: + losses: dict[str, object] = {} + gradients: dict[str, object] = {} + for reduction, builder in builders.items(): + value, loss = builder() + objective = loss.sum() if loss.ndim else loss + objective.backward() + losses[reduction] = loss.detach().clone() + gradients[reduction] = value.grad.detach().clone() + recorder.record("losses", losses) + recorder.record("input_gradients", gradients) + + +def _mse(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + source = as_profile_tensor(context, arrays["mlp_input"][:, :12]) + target = source.detach() * 0.7 - 0.15 + + def build(reduction: str) -> tuple[object, object]: + value = source.detach().clone().requires_grad_(True) + return value, functional.mse_loss(value, target, reduction=reduction) + + _record_reductions( + recorder, + {reduction: lambda reduction=reduction: build(reduction) for reduction in ("none", "mean", "sum")}, + ) + + +def _cross_entropy(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch.nn.functional as functional + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + source = as_profile_tensor(context, arrays["mlp_input"][:, :5]) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=None) + + def build(reduction: str) -> tuple[object, object]: + value = source.detach().clone().requires_grad_(True) + return value, functional.cross_entropy(value, labels, reduction=reduction) + + _record_reductions( + recorder, + {reduction: lambda reduction=reduction: build(reduction) for reduction in ("none", "mean", "sum")}, + ) + + +def _binary_cross_entropy_with_logits( + context: CaseContext, recorder: ObservationRecorder +) -> None: + import torch.nn.functional as functional + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + source = as_profile_tensor(context, arrays["mlp_input"][:, 0]) + target = as_profile_tensor(context, arrays["mlp_labels"]).to(dtype=context.torch_dtype()) + + def build(reduction: str) -> tuple[object, object]: + value = source.detach().clone().requires_grad_(True) + return value, functional.binary_cross_entropy_with_logits(value, target, reduction=reduction) + + _record_reductions( + recorder, + {reduction: lambda reduction=reduction: build(reduction) for reduction in ("none", "mean", "sum")}, + ) + + +def _kl_divergence(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + import torch.nn.functional as functional + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + source_logits = as_profile_tensor(context, arrays["mlp_input"][:, :7]) + target = torch.softmax(source_logits.detach() * 0.8 + 0.1, dim=-1) + + def build(reduction: str) -> tuple[object, object]: + value = source_logits.detach().clone().requires_grad_(True) + log_probabilities = torch.log_softmax(value, dim=-1) + return value, functional.kl_div(log_probabilities, target, reduction=reduction) + + _record_reductions( + recorder, + { + reduction: lambda reduction=reduction: build(reduction) + for reduction in ("none", "batchmean", "sum") + }, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "mse": _mse, + "cross_entropy": _cross_entropy, + "binary_cross_entropy_with_logits": _binary_cross_entropy_with_logits, + "kl_divergence": _kl_divergence, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one loss-function case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_nn_linear_and_conv.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_nn_linear_and_conv.py new file mode 100644 index 00000000..4e95b5b4 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_nn_linear_and_conv.py @@ -0,0 +1,167 @@ +"""Forward and backward cases for common neural-network layers.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import LEVEL_3_TESTS +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +NUMERICAL_DATASET_ID = "numerical_inputs_v1" +MODEL_DATASET_ID = "model_inputs_v1" + + +def _finish( + recorder: ObservationRecorder, + *, + module: object, + forward: dict[str, object], + loss: object, + input_gradients: dict[str, object], +) -> None: + loss.backward() + recorder.record("forward", forward) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record( + "input_gradients", + {name: value.grad.detach().clone() for name, value in input_gradients.items()}, + ) + recorder.record( + "parameter_gradients", + { + name: parameter.grad.detach().clone() + for name, parameter in module.named_parameters() + if parameter.grad is not None + }, + ) + + +def _linear(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + inputs = load_prepared_npz(context, MODEL_DATASET_ID, "block_inputs.npz") + state = load_prepared_npz(context, MODEL_DATASET_ID, "mlp_initial_state.npz") + value = as_profile_tensor(context, inputs["mlp_input"], requires_grad=True) + module = torch.nn.Linear(30, 32).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + module.weight.copy_(as_profile_tensor(context, state["layers.0.weight"])) + module.bias.copy_(as_profile_tensor(context, state["layers.0.bias"])) + output = module(value) + activation = torch.relu(output) + loss = activation.square().mean() + _finish( + recorder, + module=module, + forward={"output": output, "activation": activation}, + loss=loss, + input_gradients={"value": value}, + ) + + +def _conv_case( + context: CaseContext, + recorder: ObservationRecorder, + *, + dimension: int, +) -> None: + import torch + + arrays = load_prepared_npz(context, NUMERICAL_DATASET_ID, "convolutions.npz") + prefix = f"conv{dimension}d" + value = as_profile_tensor(context, arrays[f"{prefix}_input"], requires_grad=True) + weight = arrays[f"{prefix}_weight"] + bias = arrays[f"{prefix}_bias"] + convolution_class = getattr(torch.nn, f"Conv{dimension}d") + module = convolution_class( + in_channels=weight.shape[1], + out_channels=weight.shape[0], + kernel_size=weight.shape[2:], + padding=1, + bias=True, + ).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + module.weight.copy_(as_profile_tensor(context, weight)) + module.bias.copy_(as_profile_tensor(context, bias)) + output = module(value) + activation = torch.tanh(output) + loss = activation.square().mean() + _finish( + recorder, + module=module, + forward={"output": output, "activation": activation}, + loss=loss, + input_gradients={"value": value}, + ) + + +def _conv1d(context: CaseContext, recorder: ObservationRecorder) -> None: + _conv_case(context, recorder, dimension=1) + + +def _conv2d(context: CaseContext, recorder: ObservationRecorder) -> None: + _conv_case(context, recorder, dimension=2) + + +def _conv3d(context: CaseContext, recorder: ObservationRecorder) -> None: + _conv_case(context, recorder, dimension=3) + + +def _embedding(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + settings = LEVEL_3_TESTS["embedding"] + module = torch.nn.Embedding( + settings["num_embeddings"], settings["embedding_dim"] + ).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + values = torch.linspace( + -1.0, + 1.0, + steps=settings["num_embeddings"] * settings["embedding_dim"], + device=context.device, + dtype=context.torch_dtype(), + ).reshape_as(module.weight) + module.weight.copy_(values) + + indices = torch.arange( + settings["batch_size"] * settings["sequence_length"], + device=context.device, + dtype=torch.int64, + ).reshape(settings["batch_size"], settings["sequence_length"]) + indices = indices.remainder(settings["num_embeddings"]) + token_weights = torch.linspace( + 0.5, + 1.5, + steps=indices.numel(), + device=context.device, + dtype=context.torch_dtype(), + requires_grad=True, + ).reshape(*indices.shape, 1) + token_weights.retain_grad() + embedded = module(indices) + output = embedded * token_weights + pooled = output.mean(dim=1) + loss = pooled.square().mean() + _finish( + recorder, + module=module, + forward={"embedded": embedded, "output": output, "pooled": pooled}, + loss=loss, + input_gradients={"token_weights": token_weights}, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "linear": _linear, + "conv1d": _conv1d, + "conv2d": _conv2d, + "conv3d": _conv3d, + "embedding": _embedding, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one neural-network layer case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_normalisation.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_normalisation.py new file mode 100644 index 00000000..7df86ed4 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_normalisation.py @@ -0,0 +1,154 @@ +"""Training and evaluation cases for normalisation layers.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import LEVEL_3_TESTS +from cases.common import as_profile_tensor, clone_module_state, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def _finish( + recorder: ObservationRecorder, + *, + module: object, + forward: dict[str, object], + loss: object, + value: object, +) -> None: + loss.backward() + recorder.record("forward", forward) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record("input_gradients", {"value": value.grad.detach().clone()}) + recorder.record( + "parameter_gradients", + { + name: parameter.grad.detach().clone() + for name, parameter in module.named_parameters() + if parameter.grad is not None + }, + ) + recorder.record("module_state", clone_module_state(module)) + + +def _attention_input(context: CaseContext) -> object: + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + return as_profile_tensor(context, arrays["attention_input"]) + + +def _batch_norm_training(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + settings = LEVEL_3_TESTS["normalisation"] + value = _attention_input(context).transpose(1, 2).contiguous().requires_grad_(True) + module = torch.nn.BatchNorm1d( + value.shape[1], + eps=settings["epsilon"], + momentum=settings["batch_norm_momentum"], + ).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + module.weight.copy_(torch.linspace(0.8, 1.2, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.bias.copy_(torch.linspace(-0.1, 0.1, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.train() + first = module(value) + second = module(value * 0.75 + 0.1) + loss = first.square().mean() + second.abs().mean() + _finish( + recorder, + module=module, + forward={"first": first, "second": second}, + loss=loss, + value=value, + ) + + +def _batch_norm_evaluation(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + settings = LEVEL_3_TESTS["normalisation"] + value = _attention_input(context).transpose(1, 2).contiguous().requires_grad_(True) + module = torch.nn.BatchNorm1d( + value.shape[1], + eps=settings["epsilon"], + momentum=settings["batch_norm_momentum"], + ).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + module.weight.copy_(torch.linspace(0.9, 1.1, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.bias.copy_(torch.linspace(-0.05, 0.05, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.running_mean.copy_(torch.linspace(-0.2, 0.2, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.running_var.copy_(torch.linspace(0.7, 1.3, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.eval() + output = module(value) + loss = output.square().mean() + _finish( + recorder, + module=module, + forward={"output": output}, + loss=loss, + value=value, + ) + + +def _layer_norm(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + settings = LEVEL_3_TESTS["normalisation"] + value = _attention_input(context).requires_grad_(True) + module = torch.nn.LayerNorm(value.shape[-1], eps=settings["epsilon"]).to( + device=context.device, dtype=context.torch_dtype() + ) + with torch.no_grad(): + module.weight.copy_(torch.linspace(0.85, 1.15, value.shape[-1], device=context.device, dtype=context.torch_dtype())) + module.bias.copy_(torch.linspace(-0.08, 0.08, value.shape[-1], device=context.device, dtype=context.torch_dtype())) + output = module(value) + loss = output.square().mean() + _finish( + recorder, + module=module, + forward={"output": output, "feature_means": output.mean(dim=-1)}, + loss=loss, + value=value, + ) + + +def _group_norm(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + settings = LEVEL_3_TESTS["normalisation"] + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["cnn_input"]).repeat(1, 8, 1, 1) + value.requires_grad_(True) + module = torch.nn.GroupNorm( + settings["group_norm_groups"], + value.shape[1], + eps=settings["epsilon"], + ).to(device=context.device, dtype=context.torch_dtype()) + with torch.no_grad(): + module.weight.copy_(torch.linspace(0.9, 1.1, value.shape[1], device=context.device, dtype=context.torch_dtype())) + module.bias.copy_(torch.linspace(-0.05, 0.05, value.shape[1], device=context.device, dtype=context.torch_dtype())) + output = module(value) + loss = output.abs().mean() + _finish( + recorder, + module=module, + forward={"output": output, "channel_means": output.mean(dim=(0, 2, 3))}, + loss=loss, + value=value, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "batch_norm_training": _batch_norm_training, + "batch_norm_evaluation": _batch_norm_evaluation, + "layer_norm": _layer_norm, + "group_norm": _group_norm, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one normalisation case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_optimizer_adamw.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_optimizer_adamw.py new file mode 100644 index 00000000..8e83045b --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_optimizer_adamw.py @@ -0,0 +1,81 @@ +"""Short deterministic optimisation runs for AdamW variants.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import LEVEL_3_TESTS +from cases.common import ( + as_profile_tensor, + build_mlp, + clone_named_gradients, + clone_named_parameters, + flatten_optimizer_state, + load_module_state, + load_prepared_npz, + module_to_profile, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def _run_optimizer( + context: CaseContext, + recorder: ObservationRecorder, + *, + settings: dict[str, object], +) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["mlp_input"]) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + model = module_to_profile(context, build_mlp()) + load_module_state(context, model, "mlp_initial_state.npz") + optimizer = torch.optim.AdamW(model.parameters(), **settings) + steps = int(LEVEL_3_TESTS["optimizer_steps"]) + + loss_values: list[float] = [] + parameter_states: dict[str, object] = {"step_0": clone_named_parameters(model)} + parameter_gradients: dict[str, object] = {} + optimizer_states: dict[str, object] = { + "step_0": flatten_optimizer_state(optimizer, model) + } + + for step in range(steps): + optimizer.zero_grad(set_to_none=True) + logits = model(value) + loss = torch.nn.functional.cross_entropy(logits, labels) + loss_values.append(float(loss.detach().cpu().item())) + loss.backward() + parameter_gradients[f"step_{step}"] = clone_named_gradients(model) + optimizer.step() + parameter_states[f"step_{step + 1}"] = clone_named_parameters(model) + optimizer_states[f"step_{step + 1}"] = flatten_optimizer_state(optimizer, model) + + with torch.no_grad(): + final_loss = torch.nn.functional.cross_entropy(model(value), labels) + loss_values.append(float(final_loss.detach().cpu().item())) + + recorder.record("loss_series", loss_values) + recorder.record("parameter_states", parameter_states) + recorder.record("parameter_gradients", parameter_gradients) + recorder.record("optimizer_states", optimizer_states) + + +def _case(context: CaseContext, recorder: ObservationRecorder) -> None: + settings = dict(LEVEL_3_TESTS["adamw_cases"][context.case_id]) + _run_optimizer(context, recorder, settings=settings) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + case_id: _case for case_id in LEVEL_3_TESTS["adamw_cases"] +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one AdamW variant selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_optimizer_sgd.py b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_optimizer_sgd.py new file mode 100644 index 00000000..3adc834a --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_3_autograd_and_learning/test_optimizer_sgd.py @@ -0,0 +1,81 @@ +"""Short deterministic optimisation runs for SGD variants.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import LEVEL_3_TESTS +from cases.common import ( + as_profile_tensor, + build_mlp, + clone_named_gradients, + clone_named_parameters, + flatten_optimizer_state, + load_module_state, + load_prepared_npz, + module_to_profile, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def _run_optimizer( + context: CaseContext, + recorder: ObservationRecorder, + *, + settings: dict[str, object], +) -> None: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["mlp_input"]) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + model = module_to_profile(context, build_mlp()) + load_module_state(context, model, "mlp_initial_state.npz") + optimizer = torch.optim.SGD(model.parameters(), **settings) + steps = int(LEVEL_3_TESTS["optimizer_steps"]) + + loss_values: list[float] = [] + parameter_states: dict[str, object] = {"step_0": clone_named_parameters(model)} + parameter_gradients: dict[str, object] = {} + optimizer_states: dict[str, object] = { + "step_0": flatten_optimizer_state(optimizer, model) + } + + for step in range(steps): + optimizer.zero_grad(set_to_none=True) + logits = model(value) + loss = torch.nn.functional.cross_entropy(logits, labels) + loss_values.append(float(loss.detach().cpu().item())) + loss.backward() + parameter_gradients[f"step_{step}"] = clone_named_gradients(model) + optimizer.step() + parameter_states[f"step_{step + 1}"] = clone_named_parameters(model) + optimizer_states[f"step_{step + 1}"] = flatten_optimizer_state(optimizer, model) + + with torch.no_grad(): + final_loss = torch.nn.functional.cross_entropy(model(value), labels) + loss_values.append(float(final_loss.detach().cpu().item())) + + recorder.record("loss_series", loss_values) + recorder.record("parameter_states", parameter_states) + recorder.record("parameter_gradients", parameter_gradients) + recorder.record("optimizer_states", optimizer_states) + + +def _case(context: CaseContext, recorder: ObservationRecorder) -> None: + settings = dict(LEVEL_3_TESTS["sgd_cases"][context.case_id]) + _run_optimizer(context, recorder, settings=settings) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + case_id: _case for case_id in LEVEL_3_TESTS["sgd_cases"] +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one SGD variant selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/README.md b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/README.md new file mode 100644 index 00000000..90e9972d --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/README.md @@ -0,0 +1,10 @@ +# Level 4 precision and execution cases + +This level checks execution modes which can change the numerical path without changing the high-level model code + +- `test_fp32_precision_modes.py` records matmul and convolution outputs under the configured strict, high and medium float32 modes +- `test_amp_fp16.py` records CUDA FP16 autocast, unscaled gradients, optimiser-step behaviour and an intentionally injected overflow +- `test_amp_bfloat16.py` records BF16 autocast on supported CPU or CUDA builds +- `test_serialisation_roundtrip.py` checks tensor, model, optimiser and complete-checkpoint save/load paths + +The precision settings and GradScaler values are centralised in `config/suite_config.py` diff --git a/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/__init__.py b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/__init__.py new file mode 100644 index 00000000..9f29b577 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/__init__.py @@ -0,0 +1 @@ +"""Level 4 precision-mode, mixed-precision and serialisation cases.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_amp_bfloat16.py b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_amp_bfloat16.py new file mode 100644 index 00000000..a8cf50fd --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_amp_bfloat16.py @@ -0,0 +1,51 @@ +"""Exercise bfloat16 autocast for forward, backward and optimiser updates.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import LEVEL_4_TESTS +from cases.common import clone_named_gradients, clone_named_parameters, run_registered_case +from cases.common.mixed_precision import build_mlp_batch +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + + +def _run(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + model, value, labels = build_mlp_batch(context) + settings = LEVEL_4_TESTS["amp"] + optimizer = torch.optim.SGD(model.parameters(), lr=float(settings["learning_rate"])) + optimizer.zero_grad(set_to_none=True) + + with context.autocast(): + logits, activations = model(value, return_activations=True) + loss = torch.nn.functional.cross_entropy(logits, labels) + loss.backward() + + input_gradients = {"input": value.grad.detach().clone()} + parameter_gradients = clone_named_gradients(model) + if context.case_id == "optimizer_step": + optimizer.step() + updated_parameters = clone_named_parameters(model) + + forward = {"logits": logits.detach().clone()} + forward.update({f"activation.{name}": item.detach().clone() for name, item in activations.items()}) + recorder.record("forward", forward) + recorder.record("loss", float(loss.detach().cpu().item())) + recorder.record("input_gradients", input_gradients) + recorder.record("parameter_gradients", parameter_gradients) + recorder.record("updated_parameters", updated_parameters) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "forward": _run, + "backward": _run, + "optimizer_step": _run, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one bfloat16 AMP scenario selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_amp_fp16.py b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_amp_fp16.py new file mode 100644 index 00000000..471af537 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_amp_fp16.py @@ -0,0 +1,85 @@ +"""Exercise CUDA float16 autocast, gradient scaling and overflow handling.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import LEVEL_4_TESTS +from cases.common import ( + clone_named_gradients, + clone_named_parameters, + run_registered_case, +) +from cases.common.mixed_precision import ( + build_mlp_batch, + make_grad_scaler, + parameters_changed, + scaler_state_record, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder, UnsupportedCase + + +def _run(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + if torch.device(context.device).type != "cuda": + raise UnsupportedCase("FP16 autocast is only exercised on CUDA in this suite") + + model, value, labels = build_mlp_batch(context) + settings = LEVEL_4_TESTS["amp"] + optimizer = torch.optim.SGD(model.parameters(), lr=float(settings["learning_rate"])) + scaler = make_grad_scaler(context) + initial_scale = float(scaler.get_scale()) + before = clone_named_parameters(model) + + optimizer.zero_grad(set_to_none=True) + with context.autocast(): + logits, activations = model(value, return_activations=True) + normal_loss = torch.nn.functional.cross_entropy(logits, labels) + + overflow_injected = context.case_id == "loss_scaler_overflow" + backward_loss = normal_loss * float("inf") if overflow_injected else normal_loss + scaler.scale(backward_loss).backward() + scaler.unscale_(optimizer) + input_gradients = {"input": value.grad.detach().clone()} + parameter_gradients = clone_named_gradients(model) + + step_requested = context.case_id in {"optimizer_step", "loss_scaler_overflow"} + if step_requested: + scaler.step(optimizer) + scaler.update() + after = clone_named_parameters(model) + changed = parameters_changed(before, after) + step_skipped = bool(step_requested and not changed) + + forward = {"logits": logits.detach().clone()} + forward.update({f"activation.{name}": item.detach().clone() for name, item in activations.items()}) + recorder.record("forward", forward) + recorder.record("loss", float(normal_loss.detach().cpu().item())) + recorder.record("input_gradients", input_gradients) + recorder.record("parameter_gradients", parameter_gradients) + recorder.record( + "scaler_state", + scaler_state_record( + scaler, + initial_scale=initial_scale, + step_requested=step_requested, + step_skipped=step_skipped, + overflow_injected=overflow_injected, + ), + ) + recorder.record("updated_parameters", after) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "forward": _run, + "backward": _run, + "optimizer_step": _run, + "loss_scaler_overflow": _run, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one float16 AMP scenario selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_fp32_precision_modes.py b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_fp32_precision_modes.py new file mode 100644 index 00000000..63aadcf3 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_fp32_precision_modes.py @@ -0,0 +1,101 @@ +"""Exercise float32 matmul and convolution under each configured precision mode.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import PRECISION_MODE_CASES +from cases.common import as_profile_tensor, load_prepared_npz, run_registered_case +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder +from pytorch_extended_tests.precision_settings import ( + apply_float32_precision, + float32_precision_record, +) + + +DATASET_ID = "numerical_inputs_v1" + + +def _case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + import torch.nn.functional as functional + + matrix_arrays = load_prepared_npz(context, DATASET_ID, "matrix_operations.npz") + convolution_arrays = load_prepared_npz(context, DATASET_ID, "convolutions.npz") + settings = dict(PRECISION_MODE_CASES[context.case_id]) + original = float32_precision_record() + + try: + apply_float32_precision( + allow_tf32=bool(settings["allow_tf32"]), + matmul_precision=str(settings["float32_matmul_precision"]), + ) + left = as_profile_tensor(context, matrix_arrays["left"], dtype=torch.float32) + right = as_profile_tensor(context, matrix_arrays["right"], dtype=torch.float32) + batch_left = as_profile_tensor(context, matrix_arrays["batch_left"], dtype=torch.float32) + batch_right = as_profile_tensor(context, matrix_arrays["batch_right"], dtype=torch.float32) + + conv_input = as_profile_tensor( + context, convolution_arrays["conv2d_input"], dtype=torch.float32 + ) + conv_weight = as_profile_tensor( + context, convolution_arrays["conv2d_weight"], dtype=torch.float32 + ) + conv_bias = as_profile_tensor( + context, convolution_arrays["conv2d_bias"], dtype=torch.float32 + ) + grouped_input = as_profile_tensor( + context, convolution_arrays["grouped_conv2d_input"], dtype=torch.float32 + ) + grouped_weight = as_profile_tensor( + context, convolution_arrays["grouped_conv2d_weight"], dtype=torch.float32 + ) + grouped_bias = as_profile_tensor( + context, convolution_arrays["grouped_conv2d_bias"], dtype=torch.float32 + ) + + matrix_results = { + "matmul": torch.matmul(left, right), + "batched_matmul": torch.matmul(batch_left, batch_right), + "linear_equivalent": functional.linear(left, right.transpose(0, 1)), + } + convolution_results = { + "conv2d": functional.conv2d(conv_input, conv_weight, conv_bias, padding=1), + "grouped_conv2d": functional.conv2d( + grouped_input, + grouped_weight, + grouped_bias, + padding=1, + groups=2, + ), + } + applied = { + "case_id": context.case_id, + "device_type": torch.device(context.device).type, + "requested_allow_tf32": bool(settings["allow_tf32"]), + "requested_float32_matmul_precision": str( + settings["float32_matmul_precision"] + ), + **float32_precision_record(), + } + + recorder.record("matrix_results", matrix_results) + recorder.record("convolution_results", convolution_results) + recorder.record("applied_settings", applied) + finally: + original_convolution_precision = original["cudnn_convolution_precision"] + apply_float32_precision( + allow_tf32=original_convolution_precision == "tf32", + matmul_precision=str(original["float32_matmul_precision"]), + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + case_id: _case for case_id in PRECISION_MODE_CASES +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one backend precision mode selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_serialisation_roundtrip.py b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_serialisation_roundtrip.py new file mode 100644 index 00000000..fa3f1415 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_4_precision_and_execution/test_serialisation_roundtrip.py @@ -0,0 +1,253 @@ +"""Save and reload tensors, models, optimiser state and complete checkpoints.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +from config.suite_config import LEVEL_4_TESTS +from cases.common import ( + as_profile_tensor, + build_mlp, + clone_module_state, + flatten_optimizer_state, + load_module_state, + load_prepared_npz, + module_to_profile, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + + +DATASET_ID = "model_inputs_v1" + + +def _load(path: Path, *, map_location: str) -> Any: + import torch + + try: + return torch.load(path, map_location=map_location, weights_only=True) + except TypeError: + # weights_only was added after some older PyTorch releases + # The saved objects here are still only tensors and basic Python values + return torch.load(path, map_location=map_location) + + +def _tensor_structure(values: Mapping[str, Any]) -> dict[str, Any]: + import torch + + structure: dict[str, Any] = {} + for name, value in values.items(): + if isinstance(value, torch.Tensor): + structure[name] = { + "kind": "tensor", + "shape": list(value.shape), + "dtype": str(value.dtype).removeprefix("torch."), + } + else: + structure[name] = {"kind": type(value).__name__, "value": value} + return structure + + +def _flatten_loaded_optimizer(optimizer: Any, model: Any) -> dict[str, Any]: + return flatten_optimizer_state(optimizer, model) + + +def _fixed_batch(context: CaseContext) -> tuple[Any, Any]: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["mlp_input"]) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + return value, labels + + +def _new_model(context: CaseContext) -> Any: + model = module_to_profile(context, build_mlp()) + load_module_state(context, model, "mlp_initial_state.npz") + return model + + +def _tensor_case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + value, labels = _fixed_batch(context) + payload = { + "input": value, + "labels": labels, + "projection": torch.arange( + value.shape[1] * 7, + device=context.device, + dtype=context.torch_dtype(), + ).reshape(value.shape[1], 7) + / 100.0, + } + path = context.temporary_directory / "tensor_bundle.pt" + torch.save(payload, path) + loaded = _load(path, map_location=context.device) + model = _new_model(context) + with torch.no_grad(): + logits = model(loaded["input"]) + projection = loaded["input"] @ loaded["projection"] + + recorder.record("structure", _tensor_structure(loaded)) + recorder.record("loaded_values", dict(loaded)) + recorder.record("post_load_forward", {"logits": logits, "projection": projection}) + + +def _model_state_case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + value, _ = _fixed_batch(context) + model = _new_model(context) + path = context.temporary_directory / "model_state.pt" + torch.save(model.state_dict(), path) + loaded_state = _load(path, map_location=context.device) + reloaded = module_to_profile(context, build_mlp()) + reloaded.load_state_dict(loaded_state, strict=True) + with torch.no_grad(): + logits = reloaded(value) + + recorder.record("structure", _tensor_structure(loaded_state)) + recorder.record("loaded_values", dict(loaded_state)) + recorder.record("post_load_forward", {"logits": logits}) + + +def _trained_model_and_optimizer(context: CaseContext) -> tuple[Any, Any, Any, Any]: + import torch + + value, labels = _fixed_batch(context) + model = _new_model(context) + settings = LEVEL_4_TESTS["serialisation"] + optimizer = torch.optim.AdamW( + model.parameters(), + lr=float(settings["learning_rate"]), + betas=tuple(settings["betas"]), + eps=float(settings["epsilon"]), + weight_decay=float(settings["weight_decay"]), + ) + optimizer.zero_grad(set_to_none=True) + logits = model(value) + loss = torch.nn.functional.cross_entropy(logits, labels) + loss.backward() + optimizer.step() + return model, optimizer, value, labels + + +def _optimizer_state_case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + model, optimizer, value, _ = _trained_model_and_optimizer(context) + path = context.temporary_directory / "optimizer_state.pt" + torch.save( + {"model": model.state_dict(), "optimizer": optimizer.state_dict()}, + path, + ) + loaded = _load(path, map_location=context.device) + reloaded_model = module_to_profile(context, build_mlp()) + reloaded_model.load_state_dict(loaded["model"], strict=True) + settings = LEVEL_4_TESTS["serialisation"] + reloaded_optimizer = torch.optim.AdamW( + reloaded_model.parameters(), + lr=float(settings["learning_rate"]), + betas=tuple(settings["betas"]), + eps=float(settings["epsilon"]), + weight_decay=float(settings["weight_decay"]), + ) + reloaded_optimizer.load_state_dict(loaded["optimizer"]) + with torch.no_grad(): + logits = reloaded_model(value) + + loaded_values = { + **{f"model.{name}": tensor for name, tensor in clone_module_state(reloaded_model).items()}, + **{ + f"optimizer.{name}": tensor + for name, tensor in _flatten_loaded_optimizer( + reloaded_optimizer, reloaded_model + ).items() + }, + } + structure = { + "top_level_keys": sorted(loaded), + "model": _tensor_structure(loaded["model"]), + "optimizer_param_group_count": len(loaded["optimizer"]["param_groups"]), + "optimizer_state_entry_count": len(loaded["optimizer"]["state"]), + } + recorder.record("structure", structure) + recorder.record("loaded_values", loaded_values) + recorder.record("post_load_forward", {"logits": logits}) + + +def _complete_checkpoint_case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + model, optimizer, value, labels = _trained_model_and_optimizer(context) + with torch.no_grad(): + checkpoint_loss = torch.nn.functional.cross_entropy(model(value), labels) + checkpoint = { + "format_version": str(LEVEL_4_TESTS["serialisation"]["checkpoint_version"]), + "step": int(LEVEL_4_TESTS["serialisation"]["checkpoint_step"]), + "model": model.state_dict(), + "optimizer": optimizer.state_dict(), + "loss": checkpoint_loss.detach(), + "cpu_rng_state": torch.get_rng_state(), + } + path = context.temporary_directory / "complete_checkpoint.pt" + torch.save(checkpoint, path) + loaded = _load(path, map_location=context.device) + + reloaded_model = module_to_profile(context, build_mlp()) + reloaded_model.load_state_dict(loaded["model"], strict=True) + settings = LEVEL_4_TESTS["serialisation"] + reloaded_optimizer = torch.optim.AdamW( + reloaded_model.parameters(), + lr=float(settings["learning_rate"]), + betas=tuple(settings["betas"]), + eps=float(settings["epsilon"]), + weight_decay=float(settings["weight_decay"]), + ) + reloaded_optimizer.load_state_dict(loaded["optimizer"]) + with torch.no_grad(): + logits = reloaded_model(value) + loss = torch.nn.functional.cross_entropy(logits, labels) + + loaded_values = { + **{f"model.{name}": tensor for name, tensor in clone_module_state(reloaded_model).items()}, + **{ + f"optimizer.{name}": tensor + for name, tensor in _flatten_loaded_optimizer( + reloaded_optimizer, reloaded_model + ).items() + }, + "checkpoint.loss": loaded["loss"], + "checkpoint.cpu_rng_state": loaded["cpu_rng_state"], + "checkpoint.step": torch.tensor( + loaded["step"], device=context.device, dtype=torch.int64 + ), + } + structure = { + "top_level_keys": sorted(loaded), + "format_version": loaded["format_version"], + "step": int(loaded["step"]), + "model": _tensor_structure(loaded["model"]), + "optimizer_param_group_count": len(loaded["optimizer"]["param_groups"]), + "optimizer_state_entry_count": len(loaded["optimizer"]["state"]), + } + recorder.record("structure", structure) + recorder.record("loaded_values", loaded_values) + recorder.record("post_load_forward", {"logits": logits, "loss": loss.detach()}) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "tensor": _tensor_case, + "model_state": _model_state_case, + "optimizer_state": _optimizer_state_case, + "complete_checkpoint": _complete_checkpoint_case, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run one serialisation round trip selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_5_composite_models/README.md b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/README.md new file mode 100644 index 00000000..6a708f09 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/README.md @@ -0,0 +1,13 @@ +# Level 5 composite models + +These cases join several of the lower-level operations into short model blocks without yet becoming full dataset workloads + +Each model starts from a generated state and uses one fixed prepared batch. The cases retain the initial activations, first gradients, parameter checkpoints and evaluation logits around two optimiser updates + +- `test_mlp_block.py` combines Linear layers, ReLU, cross-entropy, autograd and AdamW +- `test_cnn_block.py` combines convolution, ReLU, pooling, flattening, Linear layers, cross-entropy, autograd and SGD +- `test_attention_block.py` combines projections, batched matrix multiplication, masking, softmax, residual addition, LayerNorm, pooling, cross-entropy, autograd and AdamW + +The run is intentionally short. Level 5 is meant to catch interactions between components while keeping the first divergence fairly easy to locate + +The three modules expose their example functions as well as the normal catalogue dispatcher. Level 0 calls those same functions, so the quick demonstrations and Level 5 use the same model setup and optimisation path diff --git a/pytorch/pytorch_extended_tests/cases/level_5_composite_models/__init__.py b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/__init__.py new file mode 100644 index 00000000..2bb61ca9 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/__init__.py @@ -0,0 +1 @@ +"""Level 5 composite-model cases.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_attention_block.py b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_attention_block.py new file mode 100644 index 00000000..366762db --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_attention_block.py @@ -0,0 +1,61 @@ +"""Run a short deterministic optimisation path through the attention block.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import ( + as_profile_tensor, + build_attention_block, + load_module_state, + load_prepared_npz, + module_to_profile, + run_composite_block, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def run_example(context: CaseContext, recorder: ObservationRecorder) -> dict[str, object]: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["attention_input"]) + padding_mask = as_profile_tensor( + context, + arrays["attention_padding_mask"], + dtype=torch.bool, + ) + labels = as_profile_tensor(context, arrays["attention_labels"], dtype=torch.int64) + model = module_to_profile(context, build_attention_block()) + load_module_state(context, model, "attention_initial_state.npz") + + def forward(current_model: object, retain_activations: bool) -> tuple[object, dict[str, object]]: + logits, activations = current_model( + value, + padding_mask, + return_activations=True, + ) + return logits, activations if retain_activations else {} + + return run_composite_block( + context, + recorder, + model_name="attention", + model=model, + labels=labels, + forward=forward, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "forward_backward_and_updates": run_example, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the attention composite case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_cnn_block.py b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_cnn_block.py new file mode 100644 index 00000000..07955d56 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_cnn_block.py @@ -0,0 +1,52 @@ +"""Run a short deterministic optimisation path through the fixed CNN.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import ( + as_profile_tensor, + build_cnn, + load_module_state, + load_prepared_npz, + module_to_profile, + run_composite_block, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def run_example(context: CaseContext, recorder: ObservationRecorder) -> dict[str, object]: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["cnn_input"]) + labels = as_profile_tensor(context, arrays["cnn_labels"], dtype=torch.int64) + model = module_to_profile(context, build_cnn()) + load_module_state(context, model, "cnn_initial_state.npz") + + def forward(current_model: object, retain_activations: bool) -> tuple[object, dict[str, object]]: + logits, activations = current_model(value, return_activations=True) + return logits, activations if retain_activations else {} + + return run_composite_block( + context, + recorder, + model_name="cnn", + model=model, + labels=labels, + forward=forward, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "forward_backward_and_updates": run_example, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the CNN composite case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_mlp_block.py b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_mlp_block.py new file mode 100644 index 00000000..e4f7d6b7 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_5_composite_models/test_mlp_block.py @@ -0,0 +1,52 @@ +"""Run a short deterministic optimisation path through the fixed MLP.""" + +from __future__ import annotations + +from collections.abc import Callable + +from cases.common import ( + as_profile_tensor, + build_mlp, + load_module_state, + load_prepared_npz, + module_to_profile, + run_composite_block, + run_registered_case, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +DATASET_ID = "model_inputs_v1" + + +def run_example(context: CaseContext, recorder: ObservationRecorder) -> dict[str, object]: + import torch + + arrays = load_prepared_npz(context, DATASET_ID, "block_inputs.npz") + value = as_profile_tensor(context, arrays["mlp_input"]) + labels = as_profile_tensor(context, arrays["mlp_labels"], dtype=torch.int64) + model = module_to_profile(context, build_mlp()) + load_module_state(context, model, "mlp_initial_state.npz") + + def forward(current_model: object, retain_activations: bool) -> tuple[object, dict[str, object]]: + logits, activations = current_model(value, return_activations=True) + return logits, activations if retain_activations else {} + + return run_composite_block( + context, + recorder, + model_name="mlp", + model=model, + labels=labels, + forward=forward, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "forward_backward_and_updates": run_example, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the MLP composite case selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/README.md b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/README.md new file mode 100644 index 00000000..179098a6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/README.md @@ -0,0 +1,18 @@ +# Level 6 real workloads + +These cases run short but complete training jobs on the three prepared datasets + +They are deliberately step-limited rather than accuracy benchmarks. The point is to exercise a realistic chain of data loading, forward passes, losses, backward passes, optimiser updates and full evaluation while keeping the output small enough to compare between CI jobs + +Each workload records: + +- the exact source rows used by every training batch +- full evaluation logits before training and at each configured checkpoint +- training loss at every optimiser step +- checkpoint loss and accuracy values +- all gradients from the first backward pass +- the early parameter states +- optimiser and gradient-scaler state at checkpoints +- final parameters, predictions and task metrics + +The downloaded source datasets must be prepared with `datasets/generate_datasets.py` before this level can run diff --git a/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/__init__.py b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/__init__.py new file mode 100644 index 00000000..48cf3aea --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/__init__.py @@ -0,0 +1 @@ +"""Level 6 real-workload training cases.""" diff --git a/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_cnn_training_workload.py b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_cnn_training_workload.py new file mode 100644 index 00000000..79660fe7 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_cnn_training_workload.py @@ -0,0 +1,71 @@ +"""Train the fixed CNN on the prepared Fashion-MNIST subset.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import WORKLOADS +from cases.common import ( + WorkloadBatch, + as_profile_tensor, + build_cnn, + load_module_state, + load_prepared_npz, + module_to_profile, + run_registered_case, + run_training_workload, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +WORKLOAD_NAME = "image_classification" +DATASET_ID = str(WORKLOADS[WORKLOAD_NAME]["dataset_id"]) + + +def _case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + training = load_prepared_npz(context, DATASET_ID, "train.npz") + evaluation = load_prepared_npz(context, DATASET_ID, "evaluation.npz") + model = module_to_profile(context, build_cnn()) + load_module_state( + context, + model, + str(WORKLOADS[WORKLOAD_NAME]["initial_state_file"]), + ) + + def build_training_batch(rows: object) -> WorkloadBatch: + images = as_profile_tensor(context, training["images"][rows]) + labels = as_profile_tensor(context, training["labels"][rows], dtype=torch.int64) + return WorkloadBatch((images,), {}, labels) + + def build_evaluation_batch(rows: object) -> WorkloadBatch: + images = as_profile_tensor(context, evaluation["images"][rows]) + labels = as_profile_tensor(context, evaluation["labels"][rows], dtype=torch.int64) + return WorkloadBatch((images,), {}, labels) + + def forward(current_model: object, batch: WorkloadBatch) -> object: + return current_model(*batch.args, **batch.kwargs) + + run_training_workload( + context, + recorder, + workload_name=WORKLOAD_NAME, + model=model, + training_sample_count=len(training["labels"]), + evaluation_sample_count=len(evaluation["labels"]), + training_source_indices=training["source_indices"], + build_training_batch=build_training_batch, + build_evaluation_batch=build_evaluation_batch, + forward=forward, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "fashion_mnist_cnn": _case, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the image workload selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_tabular_training_workload.py b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_tabular_training_workload.py new file mode 100644 index 00000000..a7f8a7e5 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_tabular_training_workload.py @@ -0,0 +1,71 @@ +"""Train the fixed MLP on the prepared breast-cancer dataset.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import WORKLOADS +from cases.common import ( + WorkloadBatch, + as_profile_tensor, + build_mlp, + load_module_state, + load_prepared_npz, + module_to_profile, + run_registered_case, + run_training_workload, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +WORKLOAD_NAME = "tabular_classification" +DATASET_ID = str(WORKLOADS[WORKLOAD_NAME]["dataset_id"]) + + +def _case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + training = load_prepared_npz(context, DATASET_ID, "train.npz") + evaluation = load_prepared_npz(context, DATASET_ID, "evaluation.npz") + model = module_to_profile(context, build_mlp()) + load_module_state( + context, + model, + str(WORKLOADS[WORKLOAD_NAME]["initial_state_file"]), + ) + + def build_training_batch(rows: object) -> WorkloadBatch: + features = as_profile_tensor(context, training["features"][rows]) + labels = as_profile_tensor(context, training["labels"][rows], dtype=torch.int64) + return WorkloadBatch((features,), {}, labels) + + def build_evaluation_batch(rows: object) -> WorkloadBatch: + features = as_profile_tensor(context, evaluation["features"][rows]) + labels = as_profile_tensor(context, evaluation["labels"][rows], dtype=torch.int64) + return WorkloadBatch((features,), {}, labels) + + def forward(current_model: object, batch: WorkloadBatch) -> object: + return current_model(*batch.args, **batch.kwargs) + + run_training_workload( + context, + recorder, + workload_name=WORKLOAD_NAME, + model=model, + training_sample_count=len(training["labels"]), + evaluation_sample_count=len(evaluation["labels"]), + training_source_indices=training["source_indices"], + build_training_batch=build_training_batch, + build_evaluation_batch=build_evaluation_batch, + forward=forward, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "breast_cancer_mlp": _case, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the tabular workload selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_transformer_training_workload.py b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_transformer_training_workload.py new file mode 100644 index 00000000..afd22a34 --- /dev/null +++ b/pytorch/pytorch_extended_tests/cases/level_6_real_workloads/test_transformer_training_workload.py @@ -0,0 +1,77 @@ +"""Train the fixed Transformer on the prepared SMS spam dataset.""" + +from __future__ import annotations + +from collections.abc import Callable + +from config.suite_config import WORKLOADS +from cases.common import ( + WorkloadBatch, + as_profile_tensor, + build_sms_transformer, + load_module_state, + load_prepared_npz, + module_to_profile, + run_registered_case, + run_training_workload, +) +from pytorch_extended_tests.case_api import CaseContext, ObservationRecorder + +WORKLOAD_NAME = "transformer_sequence_classification" +DATASET_ID = str(WORKLOADS[WORKLOAD_NAME]["dataset_id"]) + + +def _case(context: CaseContext, recorder: ObservationRecorder) -> None: + import torch + + training = load_prepared_npz(context, DATASET_ID, "train.npz") + evaluation = load_prepared_npz(context, DATASET_ID, "evaluation.npz") + model = module_to_profile(context, build_sms_transformer()) + load_module_state( + context, + model, + str(WORKLOADS[WORKLOAD_NAME]["initial_state_file"]), + ) + + def make_batch(dataset: dict[str, object], rows: object) -> WorkloadBatch: + input_ids = as_profile_tensor(context, dataset["input_ids"][rows], dtype=torch.int64) + attention_mask = as_profile_tensor( + context, + dataset["attention_mask"][rows], + dtype=torch.bool, + ) + labels = as_profile_tensor(context, dataset["labels"][rows], dtype=torch.int64) + return WorkloadBatch((input_ids, attention_mask), {}, labels) + + def build_training_batch(rows: object) -> WorkloadBatch: + return make_batch(training, rows) + + def build_evaluation_batch(rows: object) -> WorkloadBatch: + return make_batch(evaluation, rows) + + def forward(current_model: object, batch: WorkloadBatch) -> object: + return current_model(*batch.args, **batch.kwargs) + + run_training_workload( + context, + recorder, + workload_name=WORKLOAD_NAME, + model=model, + training_sample_count=len(training["labels"]), + evaluation_sample_count=len(evaluation["labels"]), + training_source_indices=training["source_indices"], + build_training_batch=build_training_batch, + build_evaluation_batch=build_evaluation_batch, + forward=forward, + ) + + +_CASES: dict[str, Callable[[CaseContext, ObservationRecorder], None]] = { + "sms_spam_transformer": _case, +} + + +def run_case(context: CaseContext, recorder: ObservationRecorder) -> None: + """Run the Transformer workload selected by the catalogue.""" + + run_registered_case(context, recorder, _CASES) diff --git a/pytorch/pytorch_extended_tests/config/README.md b/pytorch/pytorch_extended_tests/config/README.md new file mode 100644 index 00000000..94e6d74f --- /dev/null +++ b/pytorch/pytorch_extended_tests/config/README.md @@ -0,0 +1,99 @@ +# Configuration + +The suite uses Python configuration rather than machine-specific YAML files. +They are a bit more readable. + +## Files + +- `suite_config.py` contains suite-wide execution, seed, dataset, model, precision, AMP and workload choices +- `test_catalogue.py` contains stable test, case and output IDs +- `__init__.py` exposes the small set of version and seed values used by other packages + +`test_catalogue.py` deliberately contains no numerical tolerances. CI records raw outputs only. Comparison policies will be introduced with the separate comparison harness + +## Device selection + +The default device is CUDA. Set this environment variable for the CPU reference job: + +```bash +export PYTORCH_EXTENDED_TESTS_DEVICE=cpu +``` + +The Python orchestrator validates the value against `ALLOWED_DEVICES` + +## Seeds + +`ROOT_SEED` is the only manually selected seed. Code which needs a distinct random stream should call: + +```python +from config.suite_config import derive_seed + +seed = derive_seed("workloads.tabular_classification", "training_order") +``` + +Do not use Python's built-in `hash()` to derive seeds because its output can vary between processes + +## Changing configuration + +Changing generated dataset settings requires regenerating and recommitting the prepared datasets and `dataset_manifest.json` + +Changes which alter result meaning should also update the relevant version string. For the first implementation these are all `v1` + +## Default profiles by device + +CUDA jobs run `controlled_fp32` and `amp_fp16` by default. This gives an ordinary FP32 baseline and the usual mixed-precision FP16 training path without paying for every optional precision mode in the first CI pass + +CPU jobs run `controlled_fp32` by default. Raw FP16 is deliberately CUDA-only in this suite + +FP64, raw FP16, raw BF16 and BF16 autocast remain available as explicit opt-ins: + +```bash +PYTHONPATH=src:. python -m pytorch_extended_tests.orchestrator.run_suite \ + --profiles controlled_fp32 amp_fp16 amp_bfloat16 controlled_fp64 +``` + +## Child-process environment + +`SUBPROCESS_ENVIRONMENT` fixes Python hashing, deterministic CUDA BLAS workspace behaviour and the main CPU thread-count environment variables before each isolated test process starts. These values should normally remain unchanged within a suite version + +## Implemented and enabled levels + +All seven levels are implemented: + +- `level_0_smoke_workloads` +- `level_1_core_tensor` +- `level_2_numerical_kernels` +- `level_3_autograd_and_learning` +- `level_4_precision_and_execution` +- `level_5_composite_models` +- `level_6_real_workloads` + +`EXECUTION["enabled_levels"]` contains Level 0 only by default. The quick run needs only `model_inputs_v1`, so it does not depend on the external Level 6 datasets + +## Level 3 settings + +`LEVEL_3_TESTS` keeps the embedding shape, normalisation settings and optimiser variants in one place + +The individual case files should not add their own learning rates, optimiser betas or other suite-wide constants + +## Level 4 settings + +`PRECISION_MODE_CASES` defines the strict, high and medium float32 modes + +`LEVEL_4_TESTS` holds the AMP learning rate, GradScaler values and serialisation checkpoint settings. The mixed-precision and save/load cases should not add separate local values. + +## Level 5 settings + +`BLOCK_TESTS` contains the short composite-model training length, checkpoint steps and optimiser choices. The MLP and attention block use AdamW, while the CNN uses SGD, so the level covers both optimiser paths without adding more nearly identical cases + +## Level 6 settings + +`WORKLOADS` contains the dataset, optimiser, batch-size, training-length and checkpoint choices for each real workload + +`WORKLOAD_CAPTURE` defines which early parameter states are retained. The workload helper derives the exact shuffled batch order from the root seed and records the source indices used at every step. + +## Level 0 settings + +`LEVEL_0_DEMOS` holds the CSV filename, prediction preview length and the linear example's optimiser settings + +The MLP, CNN and attention examples deliberately reuse `BLOCK_TESTS` and the public Level 5 execution functions. This means the quick demonstrations and the detailed composite tests use the same calculations. diff --git a/pytorch/pytorch_extended_tests/config/__init__.py b/pytorch/pytorch_extended_tests/config/__init__.py new file mode 100644 index 00000000..3f61868d --- /dev/null +++ b/pytorch/pytorch_extended_tests/config/__init__.py @@ -0,0 +1,21 @@ +"""Configuration package for pytorch_extended_tests.""" + +from .suite_config import ( + CONFIG_VERSION, + RESULT_FORMAT_VERSION, + ROOT_SEED, + SUITE_NAME, + SUITE_VERSION, + TEST_CATALOGUE_VERSION, + derive_seed, +) + +__all__ = [ + "CONFIG_VERSION", + "RESULT_FORMAT_VERSION", + "ROOT_SEED", + "SUITE_NAME", + "SUITE_VERSION", + "TEST_CATALOGUE_VERSION", + "derive_seed", +] diff --git a/pytorch/pytorch_extended_tests/config/suite_config.py b/pytorch/pytorch_extended_tests/config/suite_config.py new file mode 100644 index 00000000..61e71d2d --- /dev/null +++ b/pytorch/pytorch_extended_tests/config/suite_config.py @@ -0,0 +1,670 @@ +"""Central configuration for the pytorch_extended_tests suite. + +Keep suite-wide choices here rather than spreading them through the case files. +The dataset generator imports this module as well, so it must not import PyTorch. +""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from pathlib import Path +from typing import Final + + +SUITE_NAME: Final = "pytorch_extended_tests" +SUITE_VERSION: Final = "v1" +CONFIG_VERSION: Final = "v1" +TEST_CATALOGUE_VERSION: Final = "v1" +RESULT_FORMAT_VERSION: Final = "v1" +SEED_DERIVATION_VERSION: Final = "v1" + +# This is the only root seed used by the suite +# Derive named sub-seeds with derive_seed rather than adding local constants +ROOT_SEED: Final = 42 + +REPOSITORY_ROOT: Final = Path(__file__).resolve().parents[1] +DATASETS_DIR: Final = REPOSITORY_ROOT / "datasets" +PREPARED_DATASETS_DIR: Final = DATASETS_DIR / "prepared" +DATASET_MANIFEST_PATH: Final = DATASETS_DIR / "dataset_manifest.json" +# Keep the CI path unchanged on Linux +# Use the normal temporary directory on Windows so local runs work there too +RESULTS_DIR: Final = ( + Path(tempfile.gettempdir()) / "ci_benchmarks" / "pytorch" + if os.name == "nt" + else Path("/tmp/ci_benchmarks/pytorch") +) + +DEFAULT_DEVICE: Final = "cuda" +DEVICE_ENVIRONMENT_VARIABLE: Final = "PYTORCH_EXTENDED_TESTS_DEVICE" +ALLOWED_DEVICES: Final = ("cpu", "cuda") + +# These must be set before the child Python process starts +# The values are fixed here so every CI environment gets the same behaviour +SUBPROCESS_ENVIRONMENT: Final = { + "PYTHONHASHSEED": str(ROOT_SEED), + "PYTHONUNBUFFERED": "1", + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", +} + + +# Keep the profile values as plain Python data +# The orchestrator will translate these strings into PyTorch settings +EXECUTION_PROFILES: Final = { + "controlled_fp64": { + "dtype": "float64", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "controlled_fp32": { + "dtype": "float32", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "controlled_fp16": { + "dtype": "float16", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "controlled_bfloat16": { + "dtype": "bfloat16", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "amp_fp16": { + "dtype": "float32", + "autocast_dtype": "float16", + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "amp_bfloat16": { + "dtype": "float32", + "autocast_dtype": "bfloat16", + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, +} + +DEFAULT_PROFILE_ORDER: Final = ( + "controlled_fp64", + "controlled_fp32", + "controlled_fp16", + "controlled_bfloat16", + "amp_fp16", + "amp_bfloat16", +) + +# CPU is mainly a diagnostic reference for the ordinary precision paths +# Extra CPU bfloat16 profiles can still be selected explicitly from the command line +DEFAULT_PROFILES_BY_DEVICE: Final = { + # Start with the two profiles most people will care about first + # The lower-precision CUDA profile is AMP FP16 rather than raw FP16 parameters + "cpu": ("controlled_fp32",), + "cuda": ("controlled_fp32", "amp_fp16"), +} + +LEVELS: Final = ( + "level_0_smoke_workloads", + "level_1_core_tensor", + "level_2_numerical_kernels", + "level_3_autograd_and_learning", + "level_4_precision_and_execution", + "level_5_composite_models", + "level_6_real_workloads", +) + +# All levels are implemented, but the normal CI job starts with Level 0 only +# Pass --levels explicitly once the quick demonstration results look sensible +IMPLEMENTED_LEVELS: Final = LEVELS +# DEFAULT_CI_LEVELS: Final = ("level_0_smoke_workloads",) +DEFAULT_CI_LEVELS: Final = ( + "level_0_smoke_workloads", + "level_1_core_tensor", + "level_2_numerical_kernels", + "level_3_autograd_and_learning", + "level_4_precision_and_execution", + "level_5_composite_models", +) +# levels 0-5 only need the generated data from python .\datasets\generate_datasets.py --only generated --force + +EXECUTION: Final = { + "enabled_levels": DEFAULT_CI_LEVELS, + "enabled_test_ids": (), + "disabled_test_ids": (), + "subprocess_timeout_seconds": 1_200, + "continue_after_test_file_failure": True, + "maximum_concurrent_test_files": 1, + "fail_on_missing_required_output": True, + "fail_on_unsupported_required_case": False, + "remove_existing_results": True, + # CI and normal local runs use the prepared files only + # The downloaded source archives are needed only when regenerating Level 6 data + "validate_downloaded_sources": False, + "write_catalogue_snapshot": True, +} + + +# These are the fixed model shapes used by generated initial states and case code +MODEL_ARCHITECTURES: Final = { + "linear": { + "input_features": 30, + "output_features": 2, + }, + "mlp": { + "input_features": 30, + "hidden_features": (32, 16), + "output_features": 2, + }, + "cnn": { + "channels": (1, 8, 16), + "classifier_hidden_features": 64, + "classes": 10, + }, + "attention": { + "sequence_length": 16, + "embedding_size": 32, + "heads": 4, + "classes": 2, + }, + "sms_transformer": { + "sequence_length": 64, + "vocabulary_size": 4_096, + "embedding_size": 32, + "heads": 4, + "feedforward_size": 64, + "layers": 2, + "classes": 2, + "activation": "gelu", + "dropout": 0.0, + "norm_first": False, + }, +} + + +# The generator reads this dictionary directly +# Keep its keys stable once prepared data has been committed +DATASET_GENERATION: Final = { + "breast_cancer_wisconsin": { + "evaluation_fraction": 0.2, + }, + "fashion_mnist": { + "training_samples": 4_096, + "evaluation_samples": 1_024, + }, + "sms_spam": { + "evaluation_fraction": 0.2, + "max_sequence_length": MODEL_ARCHITECTURES["sms_transformer"]["sequence_length"], + "max_vocabulary_size": MODEL_ARCHITECTURES["sms_transformer"]["vocabulary_size"], + "minimum_token_frequency": 1, + }, + "numerical_inputs": { + "vector_length": 257, + "reduction_rows": 127, + "reduction_columns": 61, + "matrix_m": 127, + "matrix_k": 61, + "matrix_n": 89, + "matrix_batch_size": 3, + }, + "model_inputs": { + "batch_size": 32, + "linear_input_features": MODEL_ARCHITECTURES["linear"]["input_features"], + "linear_output_features": MODEL_ARCHITECTURES["linear"]["output_features"], + "mlp_input_features": MODEL_ARCHITECTURES["mlp"]["input_features"], + "mlp_hidden_features": MODEL_ARCHITECTURES["mlp"]["hidden_features"], + "mlp_output_features": MODEL_ARCHITECTURES["mlp"]["output_features"], + "cnn_channels": MODEL_ARCHITECTURES["cnn"]["channels"], + "cnn_classes": MODEL_ARCHITECTURES["cnn"]["classes"], + "attention_sequence_length": MODEL_ARCHITECTURES["attention"]["sequence_length"], + "attention_embedding_size": MODEL_ARCHITECTURES["attention"]["embedding_size"], + "attention_heads": MODEL_ARCHITECTURES["attention"]["heads"], + "transformer_feedforward_size": MODEL_ARCHITECTURES["sms_transformer"]["feedforward_size"], + "transformer_layers": MODEL_ARCHITECTURES["sms_transformer"]["layers"], + }, +} + +DATASET_PATHS: Final = { + "numerical_inputs_v1": PREPARED_DATASETS_DIR / "numerical_inputs_v1", + "model_inputs_v1": PREPARED_DATASETS_DIR / "model_inputs_v1", + "breast_cancer_wisconsin_v1": PREPARED_DATASETS_DIR / "breast_cancer_wisconsin_v1", + "fashion_mnist_v1": PREPARED_DATASETS_DIR / "fashion_mnist_v1", + "sms_spam_v1": PREPARED_DATASETS_DIR / "sms_spam_v1", +} + +DATALOADER: Final = { + "num_workers": 0, + "pin_memory": False, + "persistent_workers": False, + "drop_last": False, +} + +LEVEL_3_TESTS: Final = { + "embedding": { + "num_embeddings": 23, + "embedding_dim": 8, + "batch_size": 4, + "sequence_length": 7, + }, + "normalisation": { + "epsilon": 1e-5, + "batch_norm_momentum": 0.1, + "group_norm_groups": 4, + }, + "optimizer_steps": 2, + "sgd_cases": { + "plain_sgd": { + "lr": 0.01, + "momentum": 0.0, + "weight_decay": 0.0, + "nesterov": False, + }, + "momentum": { + "lr": 0.01, + "momentum": 0.9, + "weight_decay": 0.0, + "nesterov": False, + }, + "nesterov": { + "lr": 0.01, + "momentum": 0.9, + "weight_decay": 0.0, + "nesterov": True, + }, + "weight_decay": { + "lr": 0.01, + "momentum": 0.0, + "weight_decay": 0.01, + "nesterov": False, + }, + }, + "adamw_cases": { + "default_betas": { + "lr": 0.001, + "betas": (0.9, 0.999), + "eps": 1e-8, + "weight_decay": 0.0, + "amsgrad": False, + }, + "custom_betas": { + "lr": 0.001, + "betas": (0.8, 0.95), + "eps": 1e-8, + "weight_decay": 0.0, + "amsgrad": False, + }, + "weight_decay": { + "lr": 0.001, + "betas": (0.9, 0.999), + "eps": 1e-8, + "weight_decay": 0.01, + "amsgrad": False, + }, + "amsgrad": { + "lr": 0.001, + "betas": (0.9, 0.999), + "eps": 1e-8, + "weight_decay": 0.0, + "amsgrad": True, + }, + }, +} + +AMP_GRAD_SCALER: Final = { + "initial_scale": 128.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2, +} + +LEVEL_0_DEMOS: Final = { + "summary_filename": "level_0_summary.csv", + "prediction_preview_count": 8, + "linear": { + "optimiser": "sgd", + "learning_rate": 0.02, + "momentum": 0.0, + "weight_decay": 0.0, + }, +} + +BLOCK_TESTS: Final = { + "optimisation_steps": 2, + "checkpoint_steps": (0, 1, 2), + "model_optimizers": { + "mlp": "adamw", + "cnn": "sgd", + "attention": "adamw", + }, + "sgd": { + "learning_rate": 0.01, + "momentum": 0.9, + "weight_decay": 0.0, + }, + "adamw": { + "learning_rate": 0.001, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.01, + }, +} + +WORKLOAD_CAPTURE: Final = { + "early_parameter_state_steps": (0, 1, 2), +} + +WORKLOADS: Final = { + "tabular_classification": { + "dataset_id": "breast_cancer_wisconsin_v1", + "initial_state_file": "mlp_initial_state.npz", + "training_steps": 20, + "checkpoint_steps": (0, 1, 2, 5, 10, 20), + "batch_size": 32, + "evaluation_batch_size": 256, + "shuffle_training_data": True, + "optimiser": "adamw", + "learning_rate": 0.001, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.0001, + }, + "image_classification": { + "dataset_id": "fashion_mnist_v1", + "initial_state_file": "cnn_initial_state.npz", + "training_steps": 30, + "checkpoint_steps": (0, 1, 2, 5, 10, 20, 30), + "batch_size": 64, + "evaluation_batch_size": 256, + "shuffle_training_data": True, + "optimiser": "sgd", + "learning_rate": 0.01, + "momentum": 0.9, + "weight_decay": 0.0001, + }, + "transformer_sequence_classification": { + "dataset_id": "sms_spam_v1", + "initial_state_file": "sms_transformer_initial_state.npz", + "training_steps": 30, + "checkpoint_steps": (0, 1, 2, 5, 10, 20, 30), + "batch_size": 32, + "evaluation_batch_size": 128, + "shuffle_training_data": True, + "optimiser": "adamw", + "learning_rate": 0.0005, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.01, + }, +} + +PRECISION_MODE_CASES: Final = { + "fp32_strict": { + "allow_tf32": False, + "float32_matmul_precision": "highest", + }, + "fp32_high": { + "allow_tf32": True, + "float32_matmul_precision": "high", + }, + "fp32_medium": { + "allow_tf32": True, + "float32_matmul_precision": "medium", + }, +} + +LEVEL_4_TESTS: Final = { + "amp": { + "learning_rate": 0.01, + "grad_scaler": AMP_GRAD_SCALER, + }, + "serialisation": { + "learning_rate": 0.001, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.01, + "checkpoint_version": "v1", + "checkpoint_step": 1, + }, +} + +OUTPUT_CAPTURE: Final = { + "store_tensor_payloads": True, + "store_tensor_checksums": True, + "store_first_step_gradients": True, + "store_first_step_parameter_deltas": True, + "store_optimizer_state": True, + "store_final_parameters": True, + "store_intermediate_activations_at_steps": (0,), + "store_evaluation_logits_at_checkpoints": True, + "maximum_inline_series_length": 4_096, +} + + +# Use a digest rather than hash() because Python deliberately randomises hash values +def derive_seed(*parts: str) -> int: + """Derive a stable positive seed from ROOT_SEED and a set of names.""" + + if not parts or any(not isinstance(part, str) or not part for part in parts): + raise ValueError("derive_seed requires one or more non-empty string parts") + + digest = hashlib.sha256() + digest.update(SEED_DERIVATION_VERSION.encode("ascii")) + digest.update(b"\0") + digest.update(str(ROOT_SEED).encode("ascii")) + for part in parts: + digest.update(b"\0") + digest.update(part.encode("utf-8")) + + # Stay within the range accepted cleanly by NumPy and PyTorch seed APIs + return int.from_bytes(digest.digest()[:8], "big") % (2**63 - 1) + + +def validate_suite_config() -> None: + """Check configuration relationships which would otherwise fail much later.""" + + if not isinstance(ROOT_SEED, int) or isinstance(ROOT_SEED, bool) or ROOT_SEED < 0: + raise ValueError("ROOT_SEED must be a non-negative integer") + if not RESULTS_DIR.is_absolute(): + raise ValueError("RESULTS_DIR must be an absolute path") + if DEFAULT_DEVICE not in ALLOWED_DEVICES: + raise ValueError("DEFAULT_DEVICE must be listed in ALLOWED_DEVICES") + if set(DEFAULT_PROFILE_ORDER) != set(EXECUTION_PROFILES): + raise ValueError("DEFAULT_PROFILE_ORDER must contain each execution profile once") + if len(DEFAULT_PROFILE_ORDER) != len(set(DEFAULT_PROFILE_ORDER)): + raise ValueError("DEFAULT_PROFILE_ORDER contains duplicate profiles") + if set(DEFAULT_PROFILES_BY_DEVICE) != set(ALLOWED_DEVICES): + raise ValueError("DEFAULT_PROFILES_BY_DEVICE must cover every allowed device") + for device, profile_ids in DEFAULT_PROFILES_BY_DEVICE.items(): + if not profile_ids or set(profile_ids) - set(EXECUTION_PROFILES): + raise ValueError(f"Invalid default profiles for {device}") + expected_order = tuple( + profile_id for profile_id in DEFAULT_PROFILE_ORDER if profile_id in profile_ids + ) + if tuple(profile_ids) != expected_order: + raise ValueError(f"Default profiles for {device} must use the central profile order") + enabled_levels = tuple(EXECUTION["enabled_levels"]) + if not enabled_levels: + raise ValueError("EXECUTION enabled_levels must not be empty") + if set(enabled_levels) - set(LEVELS): + raise ValueError("EXECUTION enabled_levels contains an unknown level") + expected_enabled_order = tuple(level for level in LEVELS if level in enabled_levels) + if enabled_levels != expected_enabled_order: + raise ValueError("EXECUTION enabled_levels must use the central LEVELS order") + + linear = MODEL_ARCHITECTURES["linear"] + mlp = MODEL_ARCHITECTURES["mlp"] + if int(linear["input_features"]) != int(mlp["input_features"]): + raise ValueError("The Level 0 linear and MLP inputs must use the same width") + if int(linear["output_features"]) != int(mlp["output_features"]): + raise ValueError("The Level 0 linear and MLP outputs must use the same class count") + + attention = MODEL_ARCHITECTURES["attention"] + transformer = MODEL_ARCHITECTURES["sms_transformer"] + if attention["embedding_size"] % attention["heads"] != 0: + raise ValueError("Attention embedding size must be divisible by its head count") + if transformer["embedding_size"] % transformer["heads"] != 0: + raise ValueError("Transformer embedding size must be divisible by its head count") + if int(transformer["sequence_length"]) < 2: + raise ValueError("Transformer sequence length must be at least two") + if int(transformer["vocabulary_size"]) < 8: + raise ValueError("Transformer vocabulary size must be at least eight") + if float(transformer["dropout"]) != 0.0: + raise ValueError("The fixed Transformer workload must keep dropout disabled") + if str(transformer["activation"]) not in {"relu", "gelu"}: + raise ValueError("Transformer activation must be relu or gelu") + + level_0 = LEVEL_0_DEMOS + if str(level_0["summary_filename"]) != "level_0_summary.csv": + raise ValueError("Level 0 summary filename must remain level_0_summary.csv") + if int(level_0["prediction_preview_count"]) < 1: + raise ValueError("Level 0 prediction preview count must be positive") + linear_demo = level_0["linear"] + if linear_demo["optimiser"] != "sgd": + raise ValueError("The Level 0 linear example must use SGD") + if float(linear_demo["learning_rate"]) <= 0: + raise ValueError("The Level 0 linear learning rate must be positive") + + if LEVEL_3_TESTS["optimizer_steps"] < 1: + raise ValueError("Level 3 optimizer_steps must be at least one") + if set(LEVEL_3_TESTS["sgd_cases"]) != {"plain_sgd", "momentum", "nesterov", "weight_decay"}: + raise ValueError("Level 3 SGD cases do not match the catalogue") + if set(LEVEL_3_TESTS["adamw_cases"]) != {"default_betas", "custom_betas", "weight_decay", "amsgrad"}: + raise ValueError("Level 3 AdamW cases do not match the catalogue") + + if set(PRECISION_MODE_CASES) != {"fp32_strict", "fp32_high", "fp32_medium"}: + raise ValueError("Level 4 precision-mode cases do not match the catalogue") + valid_matmul_precisions = {"highest", "high", "medium"} + for case_name, settings in PRECISION_MODE_CASES.items(): + if settings["float32_matmul_precision"] not in valid_matmul_precisions: + raise ValueError(f"Invalid float32 matmul precision for {case_name}") + if not isinstance(settings["allow_tf32"], bool): + raise ValueError(f"allow_tf32 must be Boolean for {case_name}") + + scaler = LEVEL_4_TESTS["amp"]["grad_scaler"] + if float(LEVEL_4_TESTS["amp"]["learning_rate"]) <= 0: + raise ValueError("Level 4 AMP learning rate must be positive") + if float(scaler["initial_scale"]) <= 0: + raise ValueError("Level 4 GradScaler initial scale must be positive") + if float(scaler["growth_factor"]) <= 1: + raise ValueError("Level 4 GradScaler growth factor must be greater than one") + if not 0 < float(scaler["backoff_factor"]) < 1: + raise ValueError("Level 4 GradScaler backoff factor must be between zero and one") + if int(scaler["growth_interval"]) < 1: + raise ValueError("Level 4 GradScaler growth interval must be at least one") + + block_steps = int(BLOCK_TESTS["optimisation_steps"]) + block_checkpoints = tuple(int(value) for value in BLOCK_TESTS["checkpoint_steps"]) + if block_steps < 1: + raise ValueError("Level 5 optimisation_steps must be at least one") + if tuple(sorted(set(block_checkpoints))) != block_checkpoints: + raise ValueError("Level 5 checkpoint_steps must be sorted and unique") + if block_checkpoints[0] != 0 or block_checkpoints[-1] != block_steps: + raise ValueError( + "Level 5 checkpoint_steps must start at 0 and end at optimisation_steps" + ) + expected_block_models = {"mlp", "cnn", "attention"} + if set(BLOCK_TESTS["model_optimizers"]) != expected_block_models: + raise ValueError("Level 5 model optimiser choices do not match the catalogue") + if set(BLOCK_TESTS["model_optimizers"].values()) - {"sgd", "adamw"}: + raise ValueError("Level 5 model optimisers must be sgd or adamw") + if float(BLOCK_TESTS["sgd"]["learning_rate"]) <= 0: + raise ValueError("Level 5 SGD learning rate must be positive") + if float(BLOCK_TESTS["adamw"]["learning_rate"]) <= 0: + raise ValueError("Level 5 AdamW learning rate must be positive") + + serialisation = LEVEL_4_TESTS["serialisation"] + if float(serialisation["learning_rate"]) <= 0: + raise ValueError("Level 4 serialisation learning rate must be positive") + if int(serialisation["checkpoint_step"]) < 0: + raise ValueError("Level 4 checkpoint step must be non-negative") + if not str(serialisation["checkpoint_version"]): + raise ValueError("Level 4 checkpoint version must not be empty") + + early_workload_steps = tuple( + int(value) for value in WORKLOAD_CAPTURE["early_parameter_state_steps"] + ) + if tuple(sorted(set(early_workload_steps))) != early_workload_steps: + raise ValueError("Level 6 early parameter steps must be sorted and unique") + if not early_workload_steps or early_workload_steps[0] != 0: + raise ValueError("Level 6 early parameter steps must start at zero") + + expected_workloads = { + "tabular_classification", + "image_classification", + "transformer_sequence_classification", + } + if set(WORKLOADS) != expected_workloads: + raise ValueError("Level 6 workload names do not match the catalogue") + + for workload_name, workload in WORKLOADS.items(): + steps = int(workload["training_steps"]) + checkpoints = tuple(int(value) for value in workload["checkpoint_steps"]) + if steps < 1: + raise ValueError(f"{workload_name} training_steps must be positive") + if tuple(sorted(set(checkpoints))) != checkpoints: + raise ValueError(f"{workload_name} checkpoint_steps must be sorted and unique") + if checkpoints[0] != 0 or checkpoints[-1] != steps: + raise ValueError( + f"{workload_name} checkpoint_steps must start at 0 and end at training_steps" + ) + if set(early_workload_steps) - set(checkpoints): + raise ValueError( + f"{workload_name} must include every early parameter step as a checkpoint" + ) + if workload["dataset_id"] not in DATASET_PATHS: + raise ValueError(f"{workload_name} refers to an unknown dataset") + if int(workload["batch_size"]) < 1: + raise ValueError(f"{workload_name} batch_size must be positive") + if int(workload["evaluation_batch_size"]) < 1: + raise ValueError(f"{workload_name} evaluation_batch_size must be positive") + if float(workload["learning_rate"]) <= 0: + raise ValueError(f"{workload_name} learning_rate must be positive") + if workload["optimiser"] not in {"sgd", "adamw"}: + raise ValueError(f"{workload_name} optimiser must be sgd or adamw") + + +validate_suite_config() diff --git a/pytorch/pytorch_extended_tests/config/suite_config_old.py b/pytorch/pytorch_extended_tests/config/suite_config_old.py new file mode 100644 index 00000000..c4519582 --- /dev/null +++ b/pytorch/pytorch_extended_tests/config/suite_config_old.py @@ -0,0 +1,670 @@ +"""Central configuration for the pytorch_extended_tests suite. + +Keep suite-wide choices here rather than spreading them through the case files. +The dataset generator imports this module as well, so it must not import PyTorch. +""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from pathlib import Path +from typing import Final + + +SUITE_NAME: Final = "pytorch_extended_tests" +SUITE_VERSION: Final = "v1" +CONFIG_VERSION: Final = "v1" +TEST_CATALOGUE_VERSION: Final = "v1" +RESULT_FORMAT_VERSION: Final = "v1" +SEED_DERIVATION_VERSION: Final = "v1" + +# This is the only root seed used by the suite +# Derive named sub-seeds with derive_seed rather than adding local constants +ROOT_SEED: Final = 42 + +REPOSITORY_ROOT: Final = Path(__file__).resolve().parents[1] +DATASETS_DIR: Final = REPOSITORY_ROOT / "datasets" +PREPARED_DATASETS_DIR: Final = DATASETS_DIR / "prepared" +DATASET_MANIFEST_PATH: Final = DATASETS_DIR / "dataset_manifest.json" +# Keep the CI path unchanged on Linux +# Use the normal temporary directory on Windows so local runs work there too +RESULTS_DIR: Final = ( + Path(tempfile.gettempdir()) / "ci_benchmarks" / "pytorch" + if os.name == "nt" + else Path("/tmp/ci_benchmarks/pytorch") +) + +DEFAULT_DEVICE: Final = "cuda" +DEVICE_ENVIRONMENT_VARIABLE: Final = "PYTORCH_EXTENDED_TESTS_DEVICE" +ALLOWED_DEVICES: Final = ("cpu", "cuda") + +# These must be set before the child Python process starts +# The values are fixed here so every CI environment gets the same behaviour +SUBPROCESS_ENVIRONMENT: Final = { + "PYTHONHASHSEED": str(ROOT_SEED), + "PYTHONUNBUFFERED": "1", + "CUBLAS_WORKSPACE_CONFIG": ":4096:8", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", +} + + +# Keep the profile values as plain Python data +# The orchestrator will translate these strings into PyTorch settings +EXECUTION_PROFILES: Final = { + "controlled_fp64": { + "dtype": "float64", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "controlled_fp32": { + "dtype": "float32", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "controlled_fp16": { + "dtype": "float16", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "controlled_bfloat16": { + "dtype": "bfloat16", + "autocast_dtype": None, + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "amp_fp16": { + "dtype": "float32", + "autocast_dtype": "float16", + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, + "amp_bfloat16": { + "dtype": "float32", + "autocast_dtype": "bfloat16", + "deterministic_algorithms": True, + "deterministic_warn_only": False, + "cudnn_benchmark": False, + "cudnn_deterministic": True, + "allow_tf32": False, + "float32_matmul_precision": "highest", + "cpu_threads": 1, + "interop_threads": 1, + }, +} + +DEFAULT_PROFILE_ORDER: Final = ( + "controlled_fp64", + "controlled_fp32", + "controlled_fp16", + "controlled_bfloat16", + "amp_fp16", + "amp_bfloat16", +) + +# CPU is mainly a diagnostic reference for the ordinary precision paths +# Extra CPU bfloat16 profiles can still be selected explicitly from the command line +DEFAULT_PROFILES_BY_DEVICE: Final = { + # Start with the two profiles most people will care about first + # The lower-precision CUDA profile is AMP FP16 rather than raw FP16 parameters + "cpu": ("controlled_fp32",), + "cuda": ("controlled_fp32", "amp_fp16"), +} + +LEVELS: Final = ( + "level_0_smoke_workloads", + "level_1_core_tensor", + "level_2_numerical_kernels", + "level_3_autograd_and_learning", + "level_4_precision_and_execution", + "level_5_composite_models", + "level_6_real_workloads", +) + + + +# All levels are implemented, but the normal CI job starts with Level 0 only +# Pass --levels explicitly once the quick demonstration results look sensible +IMPLEMENTED_LEVELS: Final = LEVELS +# DEFAULT_CI_LEVELS: Final = ("level_0_smoke_workloads",) +DEFAULT_CI_LEVELS: Final = ( + "level_0_smoke_workloads", + "level_1_core_tensor", + "level_2_numerical_kernels", + "level_3_autograd_and_learning", + "level_4_precision_and_execution", + "level_5_composite_models", +) +# levels 0-5 only need the generated data from python .\datasets\generate_datasets.py --only generated --force + +EXECUTION: Final = { + "enabled_levels": DEFAULT_CI_LEVELS, + "enabled_test_ids": (), + "disabled_test_ids": (), + "subprocess_timeout_seconds": 1_200, + "continue_after_test_file_failure": True, + "maximum_concurrent_test_files": 1, + "fail_on_missing_required_output": True, + "fail_on_unsupported_required_case": False, + "remove_existing_results": True, + "validate_downloaded_sources": True, + "write_catalogue_snapshot": True, +} + + +# These are the fixed model shapes used by generated initial states and case code +MODEL_ARCHITECTURES: Final = { + "linear": { + "input_features": 30, + "output_features": 2, + }, + "mlp": { + "input_features": 30, + "hidden_features": (32, 16), + "output_features": 2, + }, + "cnn": { + "channels": (1, 8, 16), + "classifier_hidden_features": 64, + "classes": 10, + }, + "attention": { + "sequence_length": 16, + "embedding_size": 32, + "heads": 4, + "classes": 2, + }, + "sms_transformer": { + "sequence_length": 64, + "vocabulary_size": 4_096, + "embedding_size": 32, + "heads": 4, + "feedforward_size": 64, + "layers": 2, + "classes": 2, + "activation": "gelu", + "dropout": 0.0, + "norm_first": False, + }, +} + + +# The generator reads this dictionary directly +# Keep its keys stable once prepared data has been committed +DATASET_GENERATION: Final = { + "breast_cancer_wisconsin": { + "evaluation_fraction": 0.2, + }, + "fashion_mnist": { + "training_samples": 4_096, + "evaluation_samples": 1_024, + }, + "sms_spam": { + "evaluation_fraction": 0.2, + "max_sequence_length": MODEL_ARCHITECTURES["sms_transformer"]["sequence_length"], + "max_vocabulary_size": MODEL_ARCHITECTURES["sms_transformer"]["vocabulary_size"], + "minimum_token_frequency": 1, + }, + "numerical_inputs": { + "vector_length": 257, + "reduction_rows": 127, + "reduction_columns": 61, + "matrix_m": 127, + "matrix_k": 61, + "matrix_n": 89, + "matrix_batch_size": 3, + }, + "model_inputs": { + "batch_size": 32, + "linear_input_features": MODEL_ARCHITECTURES["linear"]["input_features"], + "linear_output_features": MODEL_ARCHITECTURES["linear"]["output_features"], + "mlp_input_features": MODEL_ARCHITECTURES["mlp"]["input_features"], + "mlp_hidden_features": MODEL_ARCHITECTURES["mlp"]["hidden_features"], + "mlp_output_features": MODEL_ARCHITECTURES["mlp"]["output_features"], + "cnn_channels": MODEL_ARCHITECTURES["cnn"]["channels"], + "cnn_classes": MODEL_ARCHITECTURES["cnn"]["classes"], + "attention_sequence_length": MODEL_ARCHITECTURES["attention"]["sequence_length"], + "attention_embedding_size": MODEL_ARCHITECTURES["attention"]["embedding_size"], + "attention_heads": MODEL_ARCHITECTURES["attention"]["heads"], + "transformer_feedforward_size": MODEL_ARCHITECTURES["sms_transformer"]["feedforward_size"], + "transformer_layers": MODEL_ARCHITECTURES["sms_transformer"]["layers"], + }, +} + +DATASET_PATHS: Final = { + "numerical_inputs_v1": PREPARED_DATASETS_DIR / "numerical_inputs_v1", + "model_inputs_v1": PREPARED_DATASETS_DIR / "model_inputs_v1", + "breast_cancer_wisconsin_v1": PREPARED_DATASETS_DIR / "breast_cancer_wisconsin_v1", + "fashion_mnist_v1": PREPARED_DATASETS_DIR / "fashion_mnist_v1", + "sms_spam_v1": PREPARED_DATASETS_DIR / "sms_spam_v1", +} + +DATALOADER: Final = { + "num_workers": 0, + "pin_memory": False, + "persistent_workers": False, + "drop_last": False, +} + +LEVEL_3_TESTS: Final = { + "embedding": { + "num_embeddings": 23, + "embedding_dim": 8, + "batch_size": 4, + "sequence_length": 7, + }, + "normalisation": { + "epsilon": 1e-5, + "batch_norm_momentum": 0.1, + "group_norm_groups": 4, + }, + "optimizer_steps": 2, + "sgd_cases": { + "plain_sgd": { + "lr": 0.01, + "momentum": 0.0, + "weight_decay": 0.0, + "nesterov": False, + }, + "momentum": { + "lr": 0.01, + "momentum": 0.9, + "weight_decay": 0.0, + "nesterov": False, + }, + "nesterov": { + "lr": 0.01, + "momentum": 0.9, + "weight_decay": 0.0, + "nesterov": True, + }, + "weight_decay": { + "lr": 0.01, + "momentum": 0.0, + "weight_decay": 0.01, + "nesterov": False, + }, + }, + "adamw_cases": { + "default_betas": { + "lr": 0.001, + "betas": (0.9, 0.999), + "eps": 1e-8, + "weight_decay": 0.0, + "amsgrad": False, + }, + "custom_betas": { + "lr": 0.001, + "betas": (0.8, 0.95), + "eps": 1e-8, + "weight_decay": 0.0, + "amsgrad": False, + }, + "weight_decay": { + "lr": 0.001, + "betas": (0.9, 0.999), + "eps": 1e-8, + "weight_decay": 0.01, + "amsgrad": False, + }, + "amsgrad": { + "lr": 0.001, + "betas": (0.9, 0.999), + "eps": 1e-8, + "weight_decay": 0.0, + "amsgrad": True, + }, + }, +} + +AMP_GRAD_SCALER: Final = { + "initial_scale": 128.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2, +} + +LEVEL_0_DEMOS: Final = { + "summary_filename": "level_0_summary.csv", + "prediction_preview_count": 8, + "linear": { + "optimiser": "sgd", + "learning_rate": 0.02, + "momentum": 0.0, + "weight_decay": 0.0, + }, +} + +BLOCK_TESTS: Final = { + "optimisation_steps": 2, + "checkpoint_steps": (0, 1, 2), + "model_optimizers": { + "mlp": "adamw", + "cnn": "sgd", + "attention": "adamw", + }, + "sgd": { + "learning_rate": 0.01, + "momentum": 0.9, + "weight_decay": 0.0, + }, + "adamw": { + "learning_rate": 0.001, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.01, + }, +} + +WORKLOAD_CAPTURE: Final = { + "early_parameter_state_steps": (0, 1, 2), +} + +WORKLOADS: Final = { + "tabular_classification": { + "dataset_id": "breast_cancer_wisconsin_v1", + "initial_state_file": "mlp_initial_state.npz", + "training_steps": 20, + "checkpoint_steps": (0, 1, 2, 5, 10, 20), + "batch_size": 32, + "evaluation_batch_size": 256, + "shuffle_training_data": True, + "optimiser": "adamw", + "learning_rate": 0.001, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.0001, + }, + "image_classification": { + "dataset_id": "fashion_mnist_v1", + "initial_state_file": "cnn_initial_state.npz", + "training_steps": 30, + "checkpoint_steps": (0, 1, 2, 5, 10, 20, 30), + "batch_size": 64, + "evaluation_batch_size": 256, + "shuffle_training_data": True, + "optimiser": "sgd", + "learning_rate": 0.01, + "momentum": 0.9, + "weight_decay": 0.0001, + }, + "transformer_sequence_classification": { + "dataset_id": "sms_spam_v1", + "initial_state_file": "sms_transformer_initial_state.npz", + "training_steps": 30, + "checkpoint_steps": (0, 1, 2, 5, 10, 20, 30), + "batch_size": 32, + "evaluation_batch_size": 128, + "shuffle_training_data": True, + "optimiser": "adamw", + "learning_rate": 0.0005, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.01, + }, +} + +PRECISION_MODE_CASES: Final = { + "fp32_strict": { + "allow_tf32": False, + "float32_matmul_precision": "highest", + }, + "fp32_high": { + "allow_tf32": True, + "float32_matmul_precision": "high", + }, + "fp32_medium": { + "allow_tf32": True, + "float32_matmul_precision": "medium", + }, +} + +LEVEL_4_TESTS: Final = { + "amp": { + "learning_rate": 0.01, + "grad_scaler": AMP_GRAD_SCALER, + }, + "serialisation": { + "learning_rate": 0.001, + "betas": (0.9, 0.999), + "epsilon": 1e-8, + "weight_decay": 0.01, + "checkpoint_version": "v1", + "checkpoint_step": 1, + }, +} + +OUTPUT_CAPTURE: Final = { + "store_tensor_payloads": True, + "store_tensor_checksums": True, + "store_first_step_gradients": True, + "store_first_step_parameter_deltas": True, + "store_optimizer_state": True, + "store_final_parameters": True, + "store_intermediate_activations_at_steps": (0,), + "store_evaluation_logits_at_checkpoints": True, + "maximum_inline_series_length": 4_096, +} + + +# Use a digest rather than hash() because Python deliberately randomises hash values +def derive_seed(*parts: str) -> int: + """Derive a stable positive seed from ROOT_SEED and a set of names.""" + + if not parts or any(not isinstance(part, str) or not part for part in parts): + raise ValueError("derive_seed requires one or more non-empty string parts") + + digest = hashlib.sha256() + digest.update(SEED_DERIVATION_VERSION.encode("ascii")) + digest.update(b"\0") + digest.update(str(ROOT_SEED).encode("ascii")) + for part in parts: + digest.update(b"\0") + digest.update(part.encode("utf-8")) + + # Stay within the range accepted cleanly by NumPy and PyTorch seed APIs + return int.from_bytes(digest.digest()[:8], "big") % (2**63 - 1) + + +def validate_suite_config() -> None: + """Check configuration relationships which would otherwise fail much later.""" + + if not isinstance(ROOT_SEED, int) or isinstance(ROOT_SEED, bool) or ROOT_SEED < 0: + raise ValueError("ROOT_SEED must be a non-negative integer") + if not RESULTS_DIR.is_absolute(): + raise ValueError("RESULTS_DIR must be an absolute path") + if DEFAULT_DEVICE not in ALLOWED_DEVICES: + raise ValueError("DEFAULT_DEVICE must be listed in ALLOWED_DEVICES") + if set(DEFAULT_PROFILE_ORDER) != set(EXECUTION_PROFILES): + raise ValueError("DEFAULT_PROFILE_ORDER must contain each execution profile once") + if len(DEFAULT_PROFILE_ORDER) != len(set(DEFAULT_PROFILE_ORDER)): + raise ValueError("DEFAULT_PROFILE_ORDER contains duplicate profiles") + if set(DEFAULT_PROFILES_BY_DEVICE) != set(ALLOWED_DEVICES): + raise ValueError("DEFAULT_PROFILES_BY_DEVICE must cover every allowed device") + for device, profile_ids in DEFAULT_PROFILES_BY_DEVICE.items(): + if not profile_ids or set(profile_ids) - set(EXECUTION_PROFILES): + raise ValueError(f"Invalid default profiles for {device}") + expected_order = tuple( + profile_id for profile_id in DEFAULT_PROFILE_ORDER if profile_id in profile_ids + ) + if tuple(profile_ids) != expected_order: + raise ValueError(f"Default profiles for {device} must use the central profile order") + enabled_levels = tuple(EXECUTION["enabled_levels"]) + if not enabled_levels: + raise ValueError("EXECUTION enabled_levels must not be empty") + if set(enabled_levels) - set(LEVELS): + raise ValueError("EXECUTION enabled_levels contains an unknown level") + expected_enabled_order = tuple(level for level in LEVELS if level in enabled_levels) + if enabled_levels != expected_enabled_order: + raise ValueError("EXECUTION enabled_levels must use the central LEVELS order") + + linear = MODEL_ARCHITECTURES["linear"] + mlp = MODEL_ARCHITECTURES["mlp"] + if int(linear["input_features"]) != int(mlp["input_features"]): + raise ValueError("The Level 0 linear and MLP inputs must use the same width") + if int(linear["output_features"]) != int(mlp["output_features"]): + raise ValueError("The Level 0 linear and MLP outputs must use the same class count") + + attention = MODEL_ARCHITECTURES["attention"] + transformer = MODEL_ARCHITECTURES["sms_transformer"] + if attention["embedding_size"] % attention["heads"] != 0: + raise ValueError("Attention embedding size must be divisible by its head count") + if transformer["embedding_size"] % transformer["heads"] != 0: + raise ValueError("Transformer embedding size must be divisible by its head count") + if int(transformer["sequence_length"]) < 2: + raise ValueError("Transformer sequence length must be at least two") + if int(transformer["vocabulary_size"]) < 8: + raise ValueError("Transformer vocabulary size must be at least eight") + if float(transformer["dropout"]) != 0.0: + raise ValueError("The fixed Transformer workload must keep dropout disabled") + if str(transformer["activation"]) not in {"relu", "gelu"}: + raise ValueError("Transformer activation must be relu or gelu") + + level_0 = LEVEL_0_DEMOS + if str(level_0["summary_filename"]) != "level_0_summary.csv": + raise ValueError("Level 0 summary filename must remain level_0_summary.csv") + if int(level_0["prediction_preview_count"]) < 1: + raise ValueError("Level 0 prediction preview count must be positive") + linear_demo = level_0["linear"] + if linear_demo["optimiser"] != "sgd": + raise ValueError("The Level 0 linear example must use SGD") + if float(linear_demo["learning_rate"]) <= 0: + raise ValueError("The Level 0 linear learning rate must be positive") + + if LEVEL_3_TESTS["optimizer_steps"] < 1: + raise ValueError("Level 3 optimizer_steps must be at least one") + if set(LEVEL_3_TESTS["sgd_cases"]) != {"plain_sgd", "momentum", "nesterov", "weight_decay"}: + raise ValueError("Level 3 SGD cases do not match the catalogue") + if set(LEVEL_3_TESTS["adamw_cases"]) != {"default_betas", "custom_betas", "weight_decay", "amsgrad"}: + raise ValueError("Level 3 AdamW cases do not match the catalogue") + + if set(PRECISION_MODE_CASES) != {"fp32_strict", "fp32_high", "fp32_medium"}: + raise ValueError("Level 4 precision-mode cases do not match the catalogue") + valid_matmul_precisions = {"highest", "high", "medium"} + for case_name, settings in PRECISION_MODE_CASES.items(): + if settings["float32_matmul_precision"] not in valid_matmul_precisions: + raise ValueError(f"Invalid float32 matmul precision for {case_name}") + if not isinstance(settings["allow_tf32"], bool): + raise ValueError(f"allow_tf32 must be Boolean for {case_name}") + + scaler = LEVEL_4_TESTS["amp"]["grad_scaler"] + if float(LEVEL_4_TESTS["amp"]["learning_rate"]) <= 0: + raise ValueError("Level 4 AMP learning rate must be positive") + if float(scaler["initial_scale"]) <= 0: + raise ValueError("Level 4 GradScaler initial scale must be positive") + if float(scaler["growth_factor"]) <= 1: + raise ValueError("Level 4 GradScaler growth factor must be greater than one") + if not 0 < float(scaler["backoff_factor"]) < 1: + raise ValueError("Level 4 GradScaler backoff factor must be between zero and one") + if int(scaler["growth_interval"]) < 1: + raise ValueError("Level 4 GradScaler growth interval must be at least one") + + block_steps = int(BLOCK_TESTS["optimisation_steps"]) + block_checkpoints = tuple(int(value) for value in BLOCK_TESTS["checkpoint_steps"]) + if block_steps < 1: + raise ValueError("Level 5 optimisation_steps must be at least one") + if tuple(sorted(set(block_checkpoints))) != block_checkpoints: + raise ValueError("Level 5 checkpoint_steps must be sorted and unique") + if block_checkpoints[0] != 0 or block_checkpoints[-1] != block_steps: + raise ValueError( + "Level 5 checkpoint_steps must start at 0 and end at optimisation_steps" + ) + expected_block_models = {"mlp", "cnn", "attention"} + if set(BLOCK_TESTS["model_optimizers"]) != expected_block_models: + raise ValueError("Level 5 model optimiser choices do not match the catalogue") + if set(BLOCK_TESTS["model_optimizers"].values()) - {"sgd", "adamw"}: + raise ValueError("Level 5 model optimisers must be sgd or adamw") + if float(BLOCK_TESTS["sgd"]["learning_rate"]) <= 0: + raise ValueError("Level 5 SGD learning rate must be positive") + if float(BLOCK_TESTS["adamw"]["learning_rate"]) <= 0: + raise ValueError("Level 5 AdamW learning rate must be positive") + + serialisation = LEVEL_4_TESTS["serialisation"] + if float(serialisation["learning_rate"]) <= 0: + raise ValueError("Level 4 serialisation learning rate must be positive") + if int(serialisation["checkpoint_step"]) < 0: + raise ValueError("Level 4 checkpoint step must be non-negative") + if not str(serialisation["checkpoint_version"]): + raise ValueError("Level 4 checkpoint version must not be empty") + + early_workload_steps = tuple( + int(value) for value in WORKLOAD_CAPTURE["early_parameter_state_steps"] + ) + if tuple(sorted(set(early_workload_steps))) != early_workload_steps: + raise ValueError("Level 6 early parameter steps must be sorted and unique") + if not early_workload_steps or early_workload_steps[0] != 0: + raise ValueError("Level 6 early parameter steps must start at zero") + + expected_workloads = { + "tabular_classification", + "image_classification", + "transformer_sequence_classification", + } + if set(WORKLOADS) != expected_workloads: + raise ValueError("Level 6 workload names do not match the catalogue") + + for workload_name, workload in WORKLOADS.items(): + steps = int(workload["training_steps"]) + checkpoints = tuple(int(value) for value in workload["checkpoint_steps"]) + if steps < 1: + raise ValueError(f"{workload_name} training_steps must be positive") + if tuple(sorted(set(checkpoints))) != checkpoints: + raise ValueError(f"{workload_name} checkpoint_steps must be sorted and unique") + if checkpoints[0] != 0 or checkpoints[-1] != steps: + raise ValueError( + f"{workload_name} checkpoint_steps must start at 0 and end at training_steps" + ) + if set(early_workload_steps) - set(checkpoints): + raise ValueError( + f"{workload_name} must include every early parameter step as a checkpoint" + ) + if workload["dataset_id"] not in DATASET_PATHS: + raise ValueError(f"{workload_name} refers to an unknown dataset") + if int(workload["batch_size"]) < 1: + raise ValueError(f"{workload_name} batch_size must be positive") + if int(workload["evaluation_batch_size"]) < 1: + raise ValueError(f"{workload_name} evaluation_batch_size must be positive") + if float(workload["learning_rate"]) <= 0: + raise ValueError(f"{workload_name} learning_rate must be positive") + if workload["optimiser"] not in {"sgd", "adamw"}: + raise ValueError(f"{workload_name} optimiser must be sgd or adamw") + + +validate_suite_config() diff --git a/pytorch/pytorch_extended_tests/config/test_catalogue.py b/pytorch/pytorch_extended_tests/config/test_catalogue.py new file mode 100644 index 00000000..969e5bb6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/config/test_catalogue.py @@ -0,0 +1,666 @@ +"""Stable catalogue of test files, cases and expected outputs. + +The catalogue is intentionally free of numerical tolerances. CI only records raw +outputs for now, and the later comparison harness will attach policies to these +stable test, case and output IDs. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Final + +from config.suite_config import ( + DATASET_PATHS, + DEFAULT_PROFILE_ORDER, + EXECUTION_PROFILES, + LEVELS, + TEST_CATALOGUE_VERSION, +) + + +OUTPUT_KINDS: Final = { + "exact_record", + "scalar", + "tensor", + "tensor_map", + "series", + "invariant_bundle", +} +OUTPUT_IMPORTANCE: Final = {"required", "diagnostic", "informational"} + +ALL_CONTROLLED_PROFILES: Final = ( + "controlled_fp64", + "controlled_fp32", + "controlled_fp16", + "controlled_bfloat16", +) +FP32_FP64_PROFILES: Final = ("controlled_fp64", "controlled_fp32") +FP32_PROFILE: Final = ("controlled_fp32",) +TRAINING_PROFILES: Final = ("controlled_fp32", "amp_fp16", "amp_bfloat16") +AMP_FP16_PROFILE: Final = ("amp_fp16",) +AMP_BFLOAT16_PROFILE: Final = ("amp_bfloat16",) + + +@dataclass(frozen=True, slots=True) +class OutputSpec: + """One named output produced by every successful case in a test file.""" + + output_id: str + kind: str + importance: str + description: str + + +@dataclass(frozen=True, slots=True) +class TestSpec: + """Metadata needed to plan and validate one test module.""" + + test_id: str + level: str + category: str + module: str + case_ids: tuple[str, ...] + profile_ids: tuple[str, ...] + dataset_ids: tuple[str, ...] + outputs: tuple[OutputSpec, ...] + required_capabilities: tuple[str, ...] = () + unsupported_is_allowed: bool = True + + +def output( + output_id: str, + kind: str, + importance: str, + description: str, +) -> OutputSpec: + return OutputSpec(output_id, kind, importance, description) + + +STRUCTURE_AND_VALUES: Final = ( + output("structure", "exact_record", "required", "Shapes, dtypes and layout details"), + output("values", "tensor_map", "required", "Named result tensors"), +) + +FORWARD_AND_BACKWARD: Final = ( + output("forward", "tensor_map", "required", "Forward outputs and selected activations"), + output("loss", "scalar", "required", "Scalar loss used for backward"), + output("input_gradients", "tensor_map", "required", "Gradients with respect to inputs"), + output("parameter_gradients", "tensor_map", "required", "Named parameter gradients"), +) + +OPTIMISER_OUTPUTS: Final = ( + output("loss_series", "series", "required", "Loss at the initial and updated steps"), + output("parameter_states", "tensor_map", "required", "Named parameters at each step"), + output("parameter_gradients", "tensor_map", "required", "Named gradients at each step"), + output("optimizer_states", "tensor_map", "required", "Named optimiser state tensors"), +) + +BLOCK_OUTPUTS: Final = ( + output("initial_forward", "tensor_map", "required", "Initial block output and activations"), + output("loss_series", "series", "required", "Loss through the short optimisation run"), + output("first_gradients", "tensor_map", "required", "All gradients from the first backward pass"), + output("parameter_states", "tensor_map", "required", "Parameters at configured checkpoints"), + output("evaluation_outputs", "tensor_map", "required", "Fixed-batch outputs at checkpoints"), +) + +LEVEL_0_OUTPUTS: Final = ( + *BLOCK_OUTPUTS, + output( + "summary", + "exact_record", + "required", + "Small human-facing summary used to build level_0_summary.csv", + ), +) + +WORKLOAD_OUTPUTS: Final = ( + output("initial_logits", "tensor", "required", "Evaluation logits before training"), + output("initial_loss", "scalar", "required", "Evaluation loss before training"), + output("training_loss", "series", "required", "Training loss at every optimisation step"), + output("training_batch_indices", "tensor_map", "required", "Exact source rows used by each training step"), + output("checkpoint_logits", "tensor_map", "required", "Evaluation logits at configured checkpoints"), + output("checkpoint_metrics", "tensor_map", "required", "Evaluation loss and accuracy at each checkpoint"), + output("first_gradients", "tensor_map", "required", "All parameter gradients from the first step"), + output("early_parameter_states", "tensor_map", "required", "Parameters from the early checkpoints"), + output("optimizer_states", "tensor_map", "diagnostic", "Optimiser state at configured checkpoints"), + output("final_parameters", "tensor_map", "diagnostic", "Final named model parameters"), + output("final_predictions", "tensor", "diagnostic", "Final predicted classes"), + output("final_metrics", "exact_record", "required", "Loss, accuracy and sample counts"), +) + + +TEST_CATALOGUE: Final = ( + TestSpec( + test_id="demo.model_workloads", + level="level_0_smoke_workloads", + category="Quick model demonstrations", + module="cases.level_0_smoke_workloads.test_demo_workloads", + case_ids=( + "linear_classifier", + "mlp_classifier", + "cnn_classifier", + "attention_classifier", + ), + profile_ids=TRAINING_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=LEVEL_0_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="core.tensor_creation_and_dtypes", + level="level_1_core_tensor", + category="Tensor creation and dtypes", + module="cases.level_1_core_tensor.test_tensor_creation_and_dtypes", + case_ids=( + "from_numpy", + "zeros_ones_full", + "scalar_construction", + "dtype_conversion", + "device_round_trip", + "contiguous_and_non_contiguous", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=STRUCTURE_AND_VALUES, + ), + TestSpec( + test_id="core.elementwise_arithmetic", + level="level_1_core_tensor", + category="Elementwise arithmetic", + module="cases.level_1_core_tensor.test_elementwise_arithmetic", + case_ids=( + "add", + "subtract", + "multiply", + "true_divide", + "floor_divide", + "remainder", + "power", + "minimum_and_maximum", + "clamp", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("results", "tensor_map", "required", "Results for each numerical input class"),), + ), + TestSpec( + test_id="core.transcendental_functions", + level="level_1_core_tensor", + category="Mathematical functions", + module="cases.level_1_core_tensor.test_transcendental_functions", + case_ids=( + "exp_and_log", + "sqrt_and_rsqrt", + "trigonometric", + "hyperbolic", + "sigmoid_family", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("results", "tensor_map", "required", "Named function outputs"),), + ), + TestSpec( + test_id="core.indexing_and_shape", + level="level_1_core_tensor", + category="Indexing and shape operations", + module="cases.level_1_core_tensor.test_indexing_and_shape", + case_ids=( + "basic_slicing", + "advanced_indexing", + "boolean_masking", + "gather", + "scatter", + "reshape_and_view", + "transpose_and_permute", + "concatenate_and_stack", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=STRUCTURE_AND_VALUES, + ), + TestSpec( + test_id="core.type_promotion", + level="level_1_core_tensor", + category="Type promotion", + module="cases.level_1_core_tensor.test_type_promotion", + case_ids=( + "integer_and_float", + "float_widths", + "scalar_and_tensor", + "boolean_and_numeric", + "complex_and_real", + ), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=STRUCTURE_AND_VALUES, + ), + TestSpec( + test_id="kernels.reductions_and_statistics", + level="level_2_numerical_kernels", + category="Reductions and statistics", + module="cases.level_2_numerical_kernels.test_reductions_and_statistics", + case_ids=( + "sum_and_mean", + "variance_and_standard_deviation", + "minimum_and_maximum", + "cumulative_operations", + "vector_and_matrix_norms", + "cancellation_heavy_sum", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("results", "tensor_map", "required", "Scalar and tensor reduction outputs"),), + ), + TestSpec( + test_id="kernels.matrix_multiplication", + level="level_2_numerical_kernels", + category="Matrix operations", + module="cases.level_2_numerical_kernels.test_matrix_multiplication", + case_ids=( + "matrix_vector", + "matrix_matrix", + "batched_matrix_matrix", + "einsum", + "inner_and_outer", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("results", "tensor_map", "required", "Matrix operation outputs"),), + ), + TestSpec( + test_id="kernels.convolution", + level="level_2_numerical_kernels", + category="Convolution", + module="cases.level_2_numerical_kernels.test_convolution", + case_ids=("conv1d", "conv2d", "grouped_conv2d", "conv3d"), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("results", "tensor_map", "required", "Convolution outputs"),), + ), + TestSpec( + test_id="kernels.pooling", + level="level_2_numerical_kernels", + category="Pooling", + module="cases.level_2_numerical_kernels.test_pooling", + case_ids=( + "max_pool1d", + "max_pool2d", + "average_pool2d", + "adaptive_average_pool2d", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=( + output("values", "tensor_map", "required", "Pooling outputs"), + output("indices", "tensor_map", "required", "Indices returned by max pooling"), + ), + ), + TestSpec( + test_id="linalg.linear_solve", + level="level_2_numerical_kernels", + category="Linear algebra: solves", + module="cases.level_2_numerical_kernels.test_linear_solve", + case_ids=( + "well_conditioned_solve", + "ill_conditioned_solve", + "matrix_inverse", + "cholesky_solve", + ), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=( + output("solutions", "tensor_map", "required", "Calculated solutions or inverses"), + output("residuals", "tensor_map", "required", "Residuals against the original equations"), + ), + ), + TestSpec( + test_id="linalg.factorisations", + level="level_2_numerical_kernels", + category="Linear algebra: factorisations", + module="cases.level_2_numerical_kernels.test_factorisations", + case_ids=("qr", "svd", "cholesky"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("invariants", "invariant_bundle", "required", "Factors, reconstructions and residuals"),), + ), + TestSpec( + test_id="linalg.eigensystems", + level="level_2_numerical_kernels", + category="Linear algebra: eigensystems", + module="cases.level_2_numerical_kernels.test_eigensystems", + case_ids=("symmetric_distinct", "symmetric_degenerate"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("invariants", "invariant_bundle", "required", "Eigenvalues, residuals and subspace projectors"),), + ), + TestSpec( + test_id="kernels.fft", + level="level_2_numerical_kernels", + category="FFT and signal operations", + module="cases.level_2_numerical_kernels.test_fft", + case_ids=("fft_1d", "fft_2d", "real_fft", "inverse_round_trip"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=( + output("transforms", "tensor_map", "required", "Forward transform outputs"), + output("reconstructions", "tensor_map", "required", "Inverse-transform reconstructions"), + ), + ), + TestSpec( + test_id="kernels.special_functions", + level="level_2_numerical_kernels", + category="Special mathematical functions", + module="cases.level_2_numerical_kernels.test_special_functions", + case_ids=( + "erf_family", + "gamma_family", + "softmax_and_log_softmax", + "logit_and_expit", + ), + profile_ids=ALL_CONTROLLED_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=(output("results", "tensor_map", "required", "Named special-function outputs"),), + ), + TestSpec( + test_id="autograd.elementwise", + level="level_3_autograd_and_learning", + category="Autograd", + module="cases.level_3_autograd_and_learning.test_autograd_elementwise", + case_ids=("scalar_chain", "branching_graph", "reused_tensor", "reduction_graph"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=FORWARD_AND_BACKWARD, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="autograd.matrix_operations", + level="level_3_autograd_and_learning", + category="Autograd", + module="cases.level_3_autograd_and_learning.test_autograd_matrix_ops", + case_ids=("matrix_multiplication", "batched_matrix_multiplication", "convolution", "linear_solve"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1",), + outputs=FORWARD_AND_BACKWARD, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="nn.linear_and_convolution", + level="level_3_autograd_and_learning", + category="Neural-network layers", + module="cases.level_3_autograd_and_learning.test_nn_linear_and_conv", + case_ids=("linear", "conv1d", "conv2d", "conv3d", "embedding"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("numerical_inputs_v1", "model_inputs_v1"), + outputs=FORWARD_AND_BACKWARD, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="nn.normalisation", + level="level_3_autograd_and_learning", + category="Normalisation", + module="cases.level_3_autograd_and_learning.test_normalisation", + case_ids=("batch_norm_training", "batch_norm_evaluation", "layer_norm", "group_norm"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=( + *FORWARD_AND_BACKWARD, + output("module_state", "tensor_map", "required", "Named parameters and persistent normalisation state"), + ), + required_capabilities=("autograd",), + ), + TestSpec( + test_id="nn.attention", + level="level_3_autograd_and_learning", + category="Attention", + module="cases.level_3_autograd_and_learning.test_attention", + case_ids=("scaled_dot_product", "masked_scaled_dot_product", "multihead_attention"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=FORWARD_AND_BACKWARD, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="nn.losses", + level="level_3_autograd_and_learning", + category="Loss functions", + module="cases.level_3_autograd_and_learning.test_losses", + case_ids=("mse", "cross_entropy", "binary_cross_entropy_with_logits", "kl_divergence"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=( + output("losses", "tensor_map", "required", "Losses for each reduction mode"), + output("input_gradients", "tensor_map", "required", "Gradients with respect to loss inputs"), + ), + required_capabilities=("autograd",), + ), + TestSpec( + test_id="optimizers.sgd", + level="level_3_autograd_and_learning", + category="Optimisers", + module="cases.level_3_autograd_and_learning.test_optimizer_sgd", + case_ids=("plain_sgd", "momentum", "nesterov", "weight_decay"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=OPTIMISER_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="optimizers.adamw", + level="level_3_autograd_and_learning", + category="Optimisers", + module="cases.level_3_autograd_and_learning.test_optimizer_adamw", + case_ids=("default_betas", "custom_betas", "weight_decay", "amsgrad"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=OPTIMISER_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="precision.fp32_modes", + level="level_4_precision_and_execution", + category="Backend precision modes", + module="cases.level_4_precision_and_execution.test_fp32_precision_modes", + case_ids=("fp32_strict", "fp32_high", "fp32_medium"), + profile_ids=FP32_PROFILE, + dataset_ids=("numerical_inputs_v1",), + outputs=( + output("matrix_results", "tensor_map", "required", "Matrix results under each precision mode"), + output("convolution_results", "tensor_map", "required", "Convolution results under each precision mode"), + output("applied_settings", "exact_record", "required", "Precision settings applied for the case"), + ), + ), + TestSpec( + test_id="precision.amp_fp16", + level="level_4_precision_and_execution", + category="Mixed precision", + module="cases.level_4_precision_and_execution.test_amp_fp16", + case_ids=("forward", "backward", "optimizer_step", "loss_scaler_overflow"), + profile_ids=AMP_FP16_PROFILE, + dataset_ids=("model_inputs_v1",), + outputs=( + *FORWARD_AND_BACKWARD, + output("scaler_state", "exact_record", "required", "Gradient-scaler values and overflow decisions"), + output("updated_parameters", "tensor_map", "required", "Parameters after the case, including an unchanged state when no step is requested"), + ), + required_capabilities=("amp_fp16",), + ), + TestSpec( + test_id="precision.amp_bfloat16", + level="level_4_precision_and_execution", + category="Mixed precision", + module="cases.level_4_precision_and_execution.test_amp_bfloat16", + case_ids=("forward", "backward", "optimizer_step"), + profile_ids=AMP_BFLOAT16_PROFILE, + dataset_ids=("model_inputs_v1",), + outputs=( + *FORWARD_AND_BACKWARD, + output("updated_parameters", "tensor_map", "required", "Parameters after the case, including an unchanged state when no step is requested"), + ), + required_capabilities=("amp_bfloat16",), + ), + TestSpec( + test_id="execution.serialisation_roundtrip", + level="level_4_precision_and_execution", + category="Serialisation", + module="cases.level_4_precision_and_execution.test_serialisation_roundtrip", + case_ids=("tensor", "model_state", "optimizer_state", "complete_checkpoint"), + profile_ids=FP32_FP64_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=( + output("structure", "exact_record", "required", "Keys, dtypes and shapes after loading"), + output("loaded_values", "tensor_map", "required", "Loaded tensor values"), + output("post_load_forward", "tensor_map", "required", "Model outputs after loading"), + ), + required_capabilities=("serialisation",), + ), + TestSpec( + test_id="blocks.mlp", + level="level_5_composite_models", + category="MLP models", + module="cases.level_5_composite_models.test_mlp_block", + case_ids=("forward_backward_and_updates",), + profile_ids=TRAINING_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=BLOCK_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="blocks.cnn", + level="level_5_composite_models", + category="CNN models", + module="cases.level_5_composite_models.test_cnn_block", + case_ids=("forward_backward_and_updates",), + profile_ids=TRAINING_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=BLOCK_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="blocks.attention", + level="level_5_composite_models", + category="Attention models", + module="cases.level_5_composite_models.test_attention_block", + case_ids=("forward_backward_and_updates",), + profile_ids=TRAINING_PROFILES, + dataset_ids=("model_inputs_v1",), + outputs=BLOCK_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="workloads.tabular_classification", + level="level_6_real_workloads", + category="Tabular classification", + module="cases.level_6_real_workloads.test_tabular_training_workload", + case_ids=("breast_cancer_mlp",), + profile_ids=TRAINING_PROFILES, + dataset_ids=("breast_cancer_wisconsin_v1", "model_inputs_v1"), + outputs=WORKLOAD_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="workloads.image_classification", + level="level_6_real_workloads", + category="Image classification", + module="cases.level_6_real_workloads.test_cnn_training_workload", + case_ids=("fashion_mnist_cnn",), + profile_ids=TRAINING_PROFILES, + dataset_ids=("fashion_mnist_v1", "model_inputs_v1"), + outputs=WORKLOAD_OUTPUTS, + required_capabilities=("autograd",), + ), + TestSpec( + test_id="workloads.transformer_sequence_classification", + level="level_6_real_workloads", + category="Transformer sequence modelling", + module="cases.level_6_real_workloads.test_transformer_training_workload", + case_ids=("sms_spam_transformer",), + profile_ids=TRAINING_PROFILES, + dataset_ids=("sms_spam_v1", "model_inputs_v1"), + outputs=WORKLOAD_OUTPUTS, + required_capabilities=("autograd",), + ), +) + + +def validate_test_catalogue() -> None: + """Fail early when catalogue entries disagree with the central configuration.""" + + test_ids: set[str] = set() + modules: set[str] = set() + + for spec in TEST_CATALOGUE: + if spec.test_id in test_ids: + raise ValueError(f"Duplicate test ID: {spec.test_id}") + test_ids.add(spec.test_id) + + if spec.module in modules: + raise ValueError(f"Duplicate test module: {spec.module}") + modules.add(spec.module) + + if spec.level not in LEVELS: + raise ValueError(f"Unknown level for {spec.test_id}: {spec.level}") + if not spec.module.startswith("cases."): + raise ValueError(f"Test module must be in the cases package: {spec.module}") + if not spec.case_ids or len(spec.case_ids) != len(set(spec.case_ids)): + raise ValueError(f"Case IDs must be non-empty and unique for {spec.test_id}") + if not spec.profile_ids: + raise ValueError(f"No profiles configured for {spec.test_id}") + + unknown_profiles = set(spec.profile_ids) - set(EXECUTION_PROFILES) + if unknown_profiles: + raise ValueError(f"Unknown profiles for {spec.test_id}: {sorted(unknown_profiles)}") + + unknown_datasets = set(spec.dataset_ids) - set(DATASET_PATHS) + if unknown_datasets: + raise ValueError(f"Unknown datasets for {spec.test_id}: {sorted(unknown_datasets)}") + + output_ids: set[str] = set() + for output_spec in spec.outputs: + if output_spec.output_id in output_ids: + raise ValueError( + f"Duplicate output ID {output_spec.output_id!r} for {spec.test_id}" + ) + output_ids.add(output_spec.output_id) + if output_spec.kind not in OUTPUT_KINDS: + raise ValueError( + f"Unknown output kind {output_spec.kind!r} for {spec.test_id}" + ) + if output_spec.importance not in OUTPUT_IMPORTANCE: + raise ValueError( + f"Unknown output importance {output_spec.importance!r} for {spec.test_id}" + ) + + if tuple(DEFAULT_PROFILE_ORDER) != tuple(EXECUTION_PROFILES): + raise ValueError("Execution profile dictionary order must match DEFAULT_PROFILE_ORDER") + + +def catalogue_as_dict() -> dict[str, object]: + """Return a JSON-serialisable catalogue snapshot for result manifests.""" + + return { + "catalogue_version": TEST_CATALOGUE_VERSION, + "tests": [asdict(spec) for spec in TEST_CATALOGUE], + } + + +def get_test_spec(test_id: str) -> TestSpec: + """Return one catalogue entry by its stable test ID.""" + + try: + return TESTS_BY_ID[test_id] + except KeyError as exc: + raise KeyError(f"Unknown test ID: {test_id}") from exc + + +def tests_for_level(level: str) -> tuple[TestSpec, ...]: + """Return catalogue entries for one level in their declared order.""" + + if level not in LEVELS: + raise KeyError(f"Unknown test level: {level}") + return tuple(spec for spec in TEST_CATALOGUE if spec.level == level) + + +validate_test_catalogue() + +TESTS_BY_ID: Final = {spec.test_id: spec for spec in TEST_CATALOGUE} diff --git a/pytorch/pytorch_extended_tests/datasets/FASHION_MNIST_LICENSE.txt b/pytorch/pytorch_extended_tests/datasets/FASHION_MNIST_LICENSE.txt new file mode 100644 index 00000000..6bc221fc --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/FASHION_MNIST_LICENSE.txt @@ -0,0 +1,7 @@ +The MIT License (MIT) Copyright © 2017 Zalando SE, https://tech.zalando.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pytorch/pytorch_extended_tests/datasets/README.md b/pytorch/pytorch_extended_tests/datasets/README.md new file mode 100644 index 00000000..f9ef045c --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/README.md @@ -0,0 +1,216 @@ +# Datasets for `pytorch_extended_tests` + +This directory contains the fixed inputs used by the extended tests. + +CI and normal test runs use only `datasets/prepared/` and `dataset_manifest.json`. The source archives under `datasets/downloaded/` are needed only while creating or deliberately regenerating the Level 6 prepared data. After a successful full preparation, the downloaded archives can be deleted. + +The CI jobs shouldn't download or regenerate data. (though we should move the prepared data out of this repo and to some CI-readable folder) + +## Directory layout + +```text +datasets/ +├── generate_datasets.py +├── dataset_manifest.json +├── THIRD_PARTY_DATASETS.md +├── licenses/ +│ └── FASHION_MNIST_LICENSE.txt +├── downloaded/ # temporary source files, not required by CI +│ ├── breast_cancer_wisconsin/ +│ ├── fashion_mnist/ +│ └── sms_spam_collection/ +└── prepared/ + ├── numerical_inputs_v1/ + ├── model_inputs_v1/ + ├── breast_cancer_wisconsin_v1/ + ├── fashion_mnist_v1/ + └── sms_spam_v1/ +``` + +`prepared/` contains deterministic NumPy files used directly by the tests. The generator writes deterministic `.npz` archives, so running it again with the same source files, configuration and NumPy behaviour should produce the same file hashes + +`model_inputs_v1/` includes the fixed batches and initial states for the Level 0 linear, MLP, CNN and attention examples as well as the Level 6 Transformer state + +## Required suite configuration + +The generator reads the seed and generation choices from the root-relative `config/suite_config.py` file. It expects at least the following values: + +```python +SUITE_VERSION = "v1" +ROOT_SEED = 42 + +DATASET_GENERATION = { + "breast_cancer_wisconsin": { + "evaluation_fraction": 0.2, + }, + "fashion_mnist": { + "training_samples": 4096, + "evaluation_samples": 1024, + }, + "sms_spam": { + "evaluation_fraction": 0.2, + "max_sequence_length": 64, + "max_vocabulary_size": 4096, + "minimum_token_frequency": 1, + }, + "numerical_inputs": { + "vector_length": 257, + "reduction_rows": 127, + "reduction_columns": 61, + "matrix_m": 127, + "matrix_k": 61, + "matrix_n": 89, + "matrix_batch_size": 3, + }, + "model_inputs": { + "batch_size": 32, + "linear_input_features": 30, + "linear_output_features": 2, + "mlp_input_features": 30, + "mlp_hidden_features": [32, 16], + "mlp_output_features": 2, + "cnn_channels": [1, 8, 16], + "cnn_classes": 10, + "attention_sequence_length": 16, + "attention_embedding_size": 32, + "attention_heads": 4, + "transformer_feedforward_size": 64, + "transformer_layers": 2, + }, +} +``` + +These values live in `suite_config.py`. Do not add a separate seed to the generator + +## Downloaded datasets + +### Breast Cancer Wisconsin (Diagnostic) + +This is used for the tabular classification workload + +- Dataset page: +- Direct download: +- DOI: +- Licence: Creative Commons Attribution 4.0 + +Save the downloaded archive as: + +```text +datasets/downloaded/breast_cancer_wisconsin/breast_cancer_wisconsin_diagnostic.zip +``` + +### Fashion-MNIST + +This is used for the image classification workload + +- Project page: +- Licence: MIT + +Download these four official files: + +- +- +- +- + +Save them without renaming under: + +```text +datasets/downloaded/fashion_mnist/ +``` + +### SMS Spam Collection + +This is used for the small Transformer workload + +- Dataset page: +- Direct download: +- DOI: +- Licence: Creative Commons Attribution 4.0 + +Save the downloaded archive as: + +```text +datasets/downloaded/sms_spam_collection/sms_spam_collection.zip +``` + +## Preparing Levels 0–5 only + +No external downloads are needed: + +```bash +python datasets/generate_datasets.py --only generated --force +``` + +This creates the final prepared numerical/model inputs used directly by Levels 0–5 + +## Preparing Level 6 + +1. Download the three sources into the paths above +2. Run the complete generator while those files are present: + +```bash +python datasets/generate_datasets.py --force +``` + +3. Check that these directories and the manifest were updated: + +```text +datasets/prepared/breast_cancer_wisconsin_v1/ +datasets/prepared/fashion_mnist_v1/ +datasets/prepared/sms_spam_v1/ +datasets/dataset_manifest.json +``` + +4. Commit the prepared directories, manifest and third-party notices +5. Delete `datasets/downloaded/` contents if they should not be committed + +Normal Level 6 execution validates the prepared files against the manifest and does not require the downloaded archives + +To deliberately check the source archives as well, run: + +```bash +python tools/validate_setup.py \ + --levels level_6_real_workloads \ + --profiles controlled_fp32 +``` + +with `EXECUTION["validate_downloaded_sources"]` temporarily enabled, or call the validation API with source checking enabled + +## Regenerating generated inputs after deleting downloads + +This remains safe: + +```bash +python datasets/generate_datasets.py --only generated --force +``` + +The generator preserves the recorded Level 6 source provenance in the manifest when those source archives are absent and were not selected for regeneration + +Do not run the full generator after deleting the downloads. Full Level 6 regeneration needs the source files again + +## Manifest behaviour + +`dataset_manifest.json` records: + +- the configured root seed and suite version +- source URLs, licences and source hashes recorded during preparation +- SHA-256 hashes and sizes for every prepared file +- the generation timestamp + +The timestamp is informational and is not used to generate values + +## Dependencies + +The generator deliberately has a small dependency surface: + +- Python 3.10 or newer +- NumPy + +It does not require PyTorch, pandas, scikit-learn, torchvision or a Kaggle client + +## Attribution and repository use + +See [`THIRD_PARTY_DATASETS.md`](THIRD_PARTY_DATASETS.md) for the attribution, licence links, transformation notes and the SMS privacy warning + +The UCI datasets are listed as CC BY 4.0 and Fashion-MNIST is MIT licensed. Keep the notices with redistributed prepared data and check the organisation's policies before publishing third-party data diff --git a/pytorch/pytorch_extended_tests/datasets/THIRD_PARTY_DATASETS.md b/pytorch/pytorch_extended_tests/datasets/THIRD_PARTY_DATASETS.md new file mode 100644 index 00000000..1d3c39b5 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/THIRD_PARTY_DATASETS.md @@ -0,0 +1,50 @@ +# Third-party datasets + +The prepared Level 6 files are transformed copies of the datasets below + +Keep this file, `dataset_manifest.json` and the Fashion-MNIST licence notice when committing or redistributing the prepared data + +This is a record of the licences and transformations used by this repository, not legal advice + +## Breast Cancer Wisconsin (Diagnostic) + +- **Creators:** William Wolberg, Olvi Mangasarian, Nick Street and W. Street +- **Publisher:** UCI Machine Learning Repository +- **Dataset page:** https://archive.ics.uci.edu/dataset/17/breast%2Bcancer%2Bwisconsin%2Bdiagnostic +- **DOI:** https://doi.org/10.24432/C5DW2B +- **Licence:** Creative Commons Attribution 4.0 International +- **Licence text:** https://creativecommons.org/licenses/by/4.0/ + +The prepared version removes the identifier column, maps the diagnosis to integer labels, creates a fixed stratified train/evaluation split, and standardises features using the training split statistics + +Suggested citation: + +> Wolberg, W., Mangasarian, O., Street, N., & Street, W. (1993). Breast Cancer Wisconsin (Diagnostic) [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5DW2B + +## Fashion-MNIST + +- **Creator:** Zalando Research / Zalando SE +- **Project page:** https://github.com/zalandoresearch/fashion-mnist +- **Licence:** MIT +- **Licence notice:** `datasets/licenses/FASHION_MNIST_LICENSE.txt` + +The prepared version parses the original IDX files, selects fixed class-balanced train/evaluation subsets, converts images to `float32` values in `[0, 1]`, and applies no data augmentation + +The original copyright and MIT permission notice must remain with redistributed copies or substantial portions + +## SMS Spam Collection + +- **Creators:** Tiago Almeida and Jos Hidalgo +- **Publisher:** UCI Machine Learning Repository +- **Dataset page:** https://archive.ics.uci.edu/dataset/228/sms%2Bspam%2Bcollection +- **DOI:** https://doi.org/10.24432/C5CC84 +- **Licence:** Creative Commons Attribution 4.0 International +- **Licence text:** https://creativecommons.org/licenses/by/4.0/ + +The prepared version creates a fixed stratified train/evaluation split, normalises and tokenises the messages, builds the vocabulary from the training split only, and stores padded token IDs, masks and labels + +Suggested citation: + +> Almeida, T. & Hidalgo, J. (2011). SMS Spam Collection [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5CC84 + +The source contains real message text. The prepared repository files use token IDs and a generated vocabulary, but they are still derived from that text. Check the organisation's privacy and repository rules as well as the dataset licence before publishing them diff --git a/pytorch/pytorch_extended_tests/datasets/dataset_manifest.json b/pytorch/pytorch_extended_tests/datasets/dataset_manifest.json new file mode 100644 index 00000000..990788ef --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/dataset_manifest.json @@ -0,0 +1,261 @@ +{ + "generated_at_utc": "2026-07-17T04:46:33.100769+00:00", + "generated_datasets": { + "model_inputs_v1": { + "files": [ + { + "relative_path": "prepared/model_inputs_v1/attention_initial_state.npz", + "sha256": "d5ca1825a3a1f74f31ce2a6a054444febfb6707ce48b6afb16796d4e1778ecf5", + "size_bytes": 17716 + }, + { + "relative_path": "prepared/model_inputs_v1/block_inputs.npz", + "sha256": "d2bf4134c09010c8269140a8c2d8b5eeee5f4c465ae06286fcc0cf680bbc7a01", + "size_bytes": 158808 + }, + { + "relative_path": "prepared/model_inputs_v1/cnn_initial_state.npz", + "sha256": "8a73674248c4e9f9ed90eb3c1a6365367adaae0bfec9c27f42695670b2083c80", + "size_bytes": 193106 + }, + { + "relative_path": "prepared/model_inputs_v1/linear_initial_state.npz", + "sha256": "6d2dfed49c62462991af3f15a54545cfd9e3d2e6b0a57dc7001b46f16ac62415", + "size_bytes": 636 + }, + { + "relative_path": "prepared/model_inputs_v1/metadata.json", + "sha256": "0a289cbbd57c9963878bff1e85da45c3e87af0363afa3cf04ba29f1e784e3ec0", + "size_bytes": 168 + }, + { + "relative_path": "prepared/model_inputs_v1/mlp_initial_state.npz", + "sha256": "0d3bb1e93838e9cb10520bdf0a8fec1737759effd7a445594a52d1c3901d7dc8", + "size_bytes": 6764 + }, + { + "relative_path": "prepared/model_inputs_v1/sms_transformer_initial_state.npz", + "sha256": "994143d842ede80409748ec6fe787fcaaaadedcb76277cf86a62035266f5c8df", + "size_bytes": 560915 + } + ], + "kind": "generated", + "prepared_directory": "prepared/model_inputs_v1" + }, + "numerical_inputs_v1": { + "files": [ + { + "relative_path": "prepared/numerical_inputs_v1/convolutions.npz", + "sha256": "8d35185950d3cbaef794a0a56c6917e6ecd93fc79a13610572f9c9916cdb4db1", + "size_bytes": 60173 + }, + { + "relative_path": "prepared/numerical_inputs_v1/elementwise.npz", + "sha256": "d45268592e4fa6421388f83c4436c2687d88aaa23cd951c5a1e56a5ef0fc462d", + "size_bytes": 14697 + }, + { + "relative_path": "prepared/numerical_inputs_v1/fft.npz", + "sha256": "d5781938da24a05958d8a7ee309e457391b309d9e7a6e2092372c0071e93f95a", + "size_bytes": 18671 + }, + { + "relative_path": "prepared/numerical_inputs_v1/indexing.npz", + "sha256": "aea6e648c9432dcbc4b066a0025260569c6035de849db1798b703626daec22fd", + "size_bytes": 12446 + }, + { + "relative_path": "prepared/numerical_inputs_v1/linear_algebra.npz", + "sha256": "155857456b3e989ab3903e209ef8099f681f952feebcdf119d0f47f56dc8e530", + "size_bytes": 12561 + }, + { + "relative_path": "prepared/numerical_inputs_v1/matrix_operations.npz", + "sha256": "c7d13c24e9d75aa56027bc53215d66be830076a627977d4e5f76c52b29800635", + "size_bytes": 411050 + }, + { + "relative_path": "prepared/numerical_inputs_v1/metadata.json", + "sha256": "8cb57df36e45288637ce32e8698c78f0f37fc455430379c94eac3a9d472720a7", + "size_bytes": 141 + }, + { + "relative_path": "prepared/numerical_inputs_v1/reductions.npz", + "sha256": "74e2b7e78ff7694826dc8d2da313d34fbe63808ad4df636fefd4a960e84ed5b1", + "size_bytes": 146623 + }, + { + "relative_path": "prepared/numerical_inputs_v1/special_functions.npz", + "sha256": "649c985f4a5a56d6506c7f23c679328311e832b5c6d7272070aa04717f8b9181", + "size_bytes": 13036 + } + ], + "kind": "generated", + "prepared_directory": "prepared/numerical_inputs_v1" + } + }, + "manifest_version": "1", + "prepared_datasets": { + "breast_cancer_wisconsin_v1": { + "files": [ + { + "relative_path": "prepared/breast_cancer_wisconsin_v1/evaluation.npz", + "sha256": "172185a65b71599868dbcd3d7d6b5d8c88cc2a7af4f8bb3b80f8b43c8c5e64c2", + "size_bytes": 14017 + }, + { + "relative_path": "prepared/breast_cancer_wisconsin_v1/metadata.json", + "sha256": "3d949aba5541defcb2559c30393c22ace1b3c43b9e8aa666d2c467bad22218e2", + "size_bytes": 333 + }, + { + "relative_path": "prepared/breast_cancer_wisconsin_v1/preprocessing.npz", + "sha256": "aaf0f28e0bf4afbc7200bedc82578dbbc98523aef179cf69a26aece78e961815", + "size_bytes": 911 + }, + { + "relative_path": "prepared/breast_cancer_wisconsin_v1/train.npz", + "sha256": "57be34874be41c104e76047ca83edf6a85f1174e1a4427e5eeb6cfdc45654be7", + "size_bytes": 53119 + } + ], + "prepared_directory": "prepared/breast_cancer_wisconsin_v1", + "source_id": "breast_cancer_wisconsin_diagnostic" + }, + "fashion_mnist_v1": { + "files": [ + { + "relative_path": "prepared/fashion_mnist_v1/evaluation.npz", + "sha256": "4a2ebf79dec08cd641924334ae412a497a57c3878657ea4727bc7d0c1517b3dc", + "size_bytes": 662950 + }, + { + "relative_path": "prepared/fashion_mnist_v1/metadata.json", + "sha256": "f89d0169db6ccde151c4bce9fb8aa4598605401f737d67abdfdd66c32713eaa5", + "size_bytes": 252 + }, + { + "relative_path": "prepared/fashion_mnist_v1/train.npz", + "sha256": "e5977b7f09fbd25f081305ef0ae6f3c4b963edbb650195d05633ba484d71fb20", + "size_bytes": 2661946 + } + ], + "prepared_directory": "prepared/fashion_mnist_v1", + "source_id": "fashion_mnist" + }, + "sms_spam_v1": { + "files": [ + { + "relative_path": "prepared/sms_spam_v1/evaluation.npz", + "sha256": "7c812afec9f614f04d67e237589de2083fa45b35dd725c289c284595109d23d2", + "size_bytes": 49816 + }, + { + "relative_path": "prepared/sms_spam_v1/metadata.json", + "sha256": "5d00488342a0938519c82269b0c4f607c3b83f95bde546c735b7ad4ba61f74d4", + "size_bytes": 423 + }, + { + "relative_path": "prepared/sms_spam_v1/train.npz", + "sha256": "014212e58db81ede5bdbd725d98b8a0125f2020c5bc6d9f284a8c675ac73cdcd", + "size_bytes": 199241 + }, + { + "relative_path": "prepared/sms_spam_v1/vocabulary.json", + "sha256": "4a1990a6be15b48c09be49c709793403e9d9bc205ca68f1d061a0391a649b1e9", + "size_bytes": 71132 + } + ], + "prepared_directory": "prepared/sms_spam_v1", + "source_id": "sms_spam_collection" + } + }, + "root_seed": 42, + "sources": { + "breast_cancer_wisconsin_diagnostic": { + "doi": "10.24432/C5DW2B", + "download_urls": [ + "https://archive.ics.uci.edu/static/public/17/breast%2Bcancer%2Bwisconsin%2Bdiagnostic.zip" + ], + "files": [ + { + "expected_md5": null, + "md5": "97da86fb8aa67b905c88ad6c499f6bd4", + "relative_path": "downloaded/breast_cancer_wisconsin/breast_cancer_wisconsin_diagnostic.zip", + "required": true, + "sha256": "bc154869ef13f753f9e2b5a17e248cfe1ba4b6721db7c4da9f4880e40b05d3af", + "size_bytes": 51284 + } + ], + "homepage_url": "https://archive.ics.uci.edu/dataset/17/breast%2Bcancer%2Bwisconsin%2Bdiagnostic", + "kind": "download", + "licence": "CC BY 4.0" + }, + "fashion_mnist": { + "download_urls": [ + "https://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz", + "https://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz", + "https://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz", + "https://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz" + ], + "files": [ + { + "expected_md5": "8d4fb7e6c68d591d4c3dfef9ec88bf0d", + "md5": "8d4fb7e6c68d591d4c3dfef9ec88bf0d", + "relative_path": "downloaded/fashion_mnist/train-images-idx3-ubyte.gz", + "required": true, + "sha256": "3aede38d61863908ad78613f6a32ed271626dd12800ba2636569512369268a84", + "size_bytes": 26421880 + }, + { + "expected_md5": "25c81989df183df01b3e8a0aad5dffbe", + "md5": "25c81989df183df01b3e8a0aad5dffbe", + "relative_path": "downloaded/fashion_mnist/train-labels-idx1-ubyte.gz", + "required": true, + "sha256": "a04f17134ac03560a47e3764e11b92fc97de4d1bfaf8ba1a3aa29af54cc90845", + "size_bytes": 29515 + }, + { + "expected_md5": "bef4ecab320f06d8554ea6380940ec79", + "md5": "bef4ecab320f06d8554ea6380940ec79", + "relative_path": "downloaded/fashion_mnist/t10k-images-idx3-ubyte.gz", + "required": true, + "sha256": "346e55b948d973a97e58d2351dde16a484bd415d4595297633bb08f03db6a073", + "size_bytes": 4422102 + }, + { + "expected_md5": "bb300cfdad3c16e7a12a480ee83cd310", + "md5": "bb300cfdad3c16e7a12a480ee83cd310", + "relative_path": "downloaded/fashion_mnist/t10k-labels-idx1-ubyte.gz", + "required": true, + "sha256": "67da17c76eaffca5446c3361aaab5c3cd6d1c2608764d35dfb1850b086bf8dd5", + "size_bytes": 5148 + } + ], + "homepage_url": "https://github.com/zalandoresearch/fashion-mnist", + "kind": "download", + "licence": "MIT" + }, + "sms_spam_collection": { + "doi": "10.24432/C5CC84", + "download_urls": [ + "https://archive.ics.uci.edu/static/public/228/sms%2Bspam%2Bcollection.zip" + ], + "files": [ + { + "expected_md5": null, + "md5": "ab53f9571d479ee677e7b283a06a661a", + "relative_path": "downloaded/sms_spam_collection/sms_spam_collection.zip", + "required": true, + "sha256": "1587ea43e58e82b14ff1f5425c88e17f8496bfcdb67a583dbff9eefaf9963ce3", + "size_bytes": 203415 + } + ], + "homepage_url": "https://archive.ics.uci.edu/dataset/228/sms%2Bspam%2Bcollection", + "kind": "download", + "licence": "CC BY 4.0" + } + }, + "suite_name": "pytorch_extended_tests", + "suite_version": "v1" +} diff --git a/pytorch/pytorch_extended_tests/datasets/generate_datasets.py b/pytorch/pytorch_extended_tests/datasets/generate_datasets.py new file mode 100644 index 00000000..36ca0aa5 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/generate_datasets.py @@ -0,0 +1,1158 @@ +#!/usr/bin/env python3 +"""Generate and preprocess all datasets used by pytorch_extended_tests.""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import io +import json +import math +import re +import shutil +import struct +import sys +import tempfile +import unicodedata +import zipfile +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np + + +DATASETS_DIR = Path(__file__).resolve().parent +REPOSITORY_ROOT = DATASETS_DIR.parent +MANIFEST_PATH = DATASETS_DIR / "dataset_manifest.json" +DOWNLOADED_DIR = DATASETS_DIR / "downloaded" +PREPARED_DIR = DATASETS_DIR / "prepared" + +SPECIAL_TOKENS = ("[PAD]", "[UNK]", "[BOS]", "[EOS]") +TOKEN_PATTERN = re.compile(r"\w+(?:['’]\w+)*|[^\w\s]", flags=re.UNICODE) + + +class DatasetGenerationError(RuntimeError): + """Raised when source data or generation configuration is invalid.""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate deterministic inputs and preprocess downloaded datasets", + ) + parser.add_argument( + "--force", + action="store_true", + help="Replace prepared directories that already exist", + ) + parser.add_argument( + "--only", + choices=("all", "generated", "breast-cancer", "fashion-mnist", "sms-spam"), + default="all", + help="Prepare one part of the dataset tree", + ) + return parser.parse_args() + + +def load_suite_configuration() -> tuple[str, int, dict[str, Any]]: + sys.path.insert(0, str(REPOSITORY_ROOT)) + try: + from config.suite_config import DATASET_GENERATION, ROOT_SEED, SUITE_VERSION + except (ImportError, AttributeError) as exc: + raise DatasetGenerationError( + "Could not load SUITE_VERSION, ROOT_SEED and DATASET_GENERATION " + "from config/suite_config.py" + ) from exc + + if not isinstance(SUITE_VERSION, str) or not SUITE_VERSION: + raise DatasetGenerationError("SUITE_VERSION must be a non-empty string") + if not isinstance(ROOT_SEED, int) or isinstance(ROOT_SEED, bool): + raise DatasetGenerationError("ROOT_SEED must be an integer") + if not isinstance(DATASET_GENERATION, dict): + raise DatasetGenerationError("DATASET_GENERATION must be a dictionary") + + return SUITE_VERSION, ROOT_SEED, DATASET_GENERATION + + +def require_mapping(config: Mapping[str, Any], key: str) -> Mapping[str, Any]: + value = config.get(key) + if not isinstance(value, Mapping): + raise DatasetGenerationError(f"DATASET_GENERATION[{key!r}] must be a mapping") + return value + + +def require_int(config: Mapping[str, Any], key: str, minimum: int = 1) -> int: + value = config.get(key) + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise DatasetGenerationError(f"{key!r} must be an integer >= {minimum}") + return value + + +def require_float( + config: Mapping[str, Any], + key: str, + minimum: float, + maximum: float, +) -> float: + value = config.get(key) + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise DatasetGenerationError(f"{key!r} must be numeric") + result = float(value) + if not minimum < result < maximum: + raise DatasetGenerationError(f"{key!r} must be between {minimum} and {maximum}") + return result + + +def require_int_sequence( + config: Mapping[str, Any], + key: str, + expected_length: int | None = None, +) -> tuple[int, ...]: + value = config.get(key) + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise DatasetGenerationError(f"{key!r} must be a sequence of integers") + result = tuple(value) + if not result or any(not isinstance(item, int) or item < 1 for item in result): + raise DatasetGenerationError(f"{key!r} must contain positive integers") + if expected_length is not None and len(result) != expected_length: + raise DatasetGenerationError(f"{key!r} must contain {expected_length} values") + return result + + +def stable_seed(root_seed: int, *parts: str) -> int: + digest = hashlib.sha256() + digest.update(str(root_seed).encode("ascii")) + for part in parts: + digest.update(b"\0") + digest.update(part.encode("utf-8")) + return int.from_bytes(digest.digest()[:8], "big", signed=False) + + +def make_rng(root_seed: int, *parts: str) -> np.random.Generator: + return np.random.default_rng(stable_seed(root_seed, *parts)) + + +def hash_file(path: Path, algorithm: str) -> str: + digest = hashlib.new(algorithm) + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + serialised = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + path.write_text(serialised + "\n", encoding="utf-8", newline="\n") + + +def write_deterministic_npz(path: Path, arrays: Mapping[str, np.ndarray]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, mode="w") as archive: + for name in sorted(arrays): + array = np.asarray(arrays[name]) + if array.dtype.hasobject: + raise DatasetGenerationError(f"Object array {name!r} cannot be stored safely") + + buffer = io.BytesIO() + np.lib.format.write_array(buffer, array, allow_pickle=False) + member = zipfile.ZipInfo(f"{name}.npy", date_time=(1980, 1, 1, 0, 0, 0)) + member.compress_type = zipfile.ZIP_DEFLATED + member.external_attr = 0o644 << 16 + archive.writestr( + member, + buffer.getvalue(), + compress_type=zipfile.ZIP_DEFLATED, + compresslevel=9, + ) + + +def replace_directory(source: Path, target: Path, force: bool) -> None: + if target.exists(): + if not force: + raise DatasetGenerationError( + f"{target} already exists. Use --force to replace prepared data" + ) + shutil.rmtree(target) + target.parent.mkdir(parents=True, exist_ok=True) + source.replace(target) + + +def source_file(manifest: Mapping[str, Any], source_id: str, index: int = 0) -> Path: + try: + relative_path = manifest["sources"][source_id]["files"][index]["relative_path"] + except (KeyError, IndexError, TypeError) as exc: + raise DatasetGenerationError(f"Manifest source {source_id!r} is invalid") from exc + return DATASETS_DIR / relative_path + + +def require_source(path: Path, urls: Sequence[str]) -> None: + if path.is_file(): + return + links = "\n".join(f" {url}" for url in urls) + raise DatasetGenerationError( + f"Required source file is missing: {path}\nDownload it from:\n{links}" + ) + + +def verify_manifest_sources(manifest: Mapping[str, Any], selected_ids: set[str]) -> None: + for source_id in selected_ids: + source = manifest["sources"][source_id] + urls = source["download_urls"] + for file_spec in source["files"]: + path = DATASETS_DIR / file_spec["relative_path"] + require_source(path, urls) + expected_md5 = file_spec.get("expected_md5") + if expected_md5 is not None: + actual_md5 = hash_file(path, "md5") + if actual_md5.lower() != expected_md5.lower(): + raise DatasetGenerationError( + f"MD5 mismatch for {path}\n" + f"Expected {expected_md5}\n" + f"Actual {actual_md5}" + ) + + +def stratified_split_indices( + labels: np.ndarray, + evaluation_fraction: float, + rng: np.random.Generator, +) -> tuple[np.ndarray, np.ndarray]: + train_parts: list[np.ndarray] = [] + evaluation_parts: list[np.ndarray] = [] + + for label in np.unique(labels): + indices = np.flatnonzero(labels == label) + shuffled = rng.permutation(indices) + evaluation_count = max(1, int(round(len(indices) * evaluation_fraction))) + evaluation_count = min(evaluation_count, len(indices) - 1) + evaluation_parts.append(shuffled[:evaluation_count]) + train_parts.append(shuffled[evaluation_count:]) + + train = rng.permutation(np.concatenate(train_parts)).astype(np.int64) + evaluation = rng.permutation(np.concatenate(evaluation_parts)).astype(np.int64) + return train, evaluation + + +def balanced_subset_indices( + labels: np.ndarray, + total_count: int, + rng: np.random.Generator, +) -> np.ndarray: + classes = np.unique(labels) + if total_count > len(labels): + raise DatasetGenerationError( + f"Requested {total_count} samples from a dataset with {len(labels)} rows" + ) + + base_count, remainder = divmod(total_count, len(classes)) + selected: list[np.ndarray] = [] + for position, label in enumerate(classes): + class_count = base_count + int(position < remainder) + candidates = np.flatnonzero(labels == label) + if class_count > len(candidates): + raise DatasetGenerationError( + f"Not enough rows for class {label}: requested {class_count}, found {len(candidates)}" + ) + selected.append(rng.permutation(candidates)[:class_count]) + + return rng.permutation(np.concatenate(selected)).astype(np.int64) + + +def xavier_uniform( + rng: np.random.Generator, + shape: Sequence[int], + fan_in: int, + fan_out: int, +) -> np.ndarray: + limit = math.sqrt(6.0 / float(fan_in + fan_out)) + return rng.uniform(-limit, limit, size=shape).astype(np.float32) + + +def linear_state( + rng: np.random.Generator, + prefix: str, + input_features: int, + output_features: int, +) -> dict[str, np.ndarray]: + return { + f"{prefix}.weight": xavier_uniform( + rng, + (output_features, input_features), + input_features, + output_features, + ), + f"{prefix}.bias": np.zeros(output_features, dtype=np.float32), + } + + +def conv2d_state( + rng: np.random.Generator, + prefix: str, + input_channels: int, + output_channels: int, + kernel_size: int, +) -> dict[str, np.ndarray]: + fan_in = input_channels * kernel_size * kernel_size + fan_out = output_channels * kernel_size * kernel_size + return { + f"{prefix}.weight": xavier_uniform( + rng, + (output_channels, input_channels, kernel_size, kernel_size), + fan_in, + fan_out, + ), + f"{prefix}.bias": np.zeros(output_channels, dtype=np.float32), + } + + +def generate_numerical_inputs( + target: Path, + root_seed: int, + config: Mapping[str, Any], +) -> None: + vector_length = require_int(config, "vector_length", minimum=16) + reduction_rows = require_int(config, "reduction_rows", minimum=3) + reduction_columns = require_int(config, "reduction_columns", minimum=3) + matrix_m = require_int(config, "matrix_m", minimum=3) + matrix_k = require_int(config, "matrix_k", minimum=3) + matrix_n = require_int(config, "matrix_n", minimum=3) + matrix_batch_size = require_int(config, "matrix_batch_size", minimum=2) + + target.mkdir(parents=True, exist_ok=True) + + rng = make_rng(root_seed, "numerical_inputs", "elementwise") + signs = np.where(np.arange(vector_length) % 2 == 0, 1.0, -1.0) + elementwise = { + "ordinary": rng.normal(0.0, 2.0, size=vector_length).astype(np.float64), + "near_zero": ( + signs * np.logspace(-16, -2, num=vector_length, dtype=np.float64) + ), + "large_magnitude": ( + signs * np.logspace(2, 12, num=vector_length, dtype=np.float64) + ), + "mixed_sign": rng.uniform(-10.0, 10.0, size=vector_length).astype(np.float64), + "positive": rng.uniform(1e-6, 20.0, size=vector_length).astype(np.float64), + "unit_interval": rng.uniform(-0.999, 0.999, size=vector_length).astype(np.float64), + "broadcast_left": rng.normal(size=(7, 1, 13)).astype(np.float64), + "broadcast_right": rng.normal(size=(1, 11, 1)).astype(np.float64), + "special_values": np.array( + [0.0, -0.0, np.inf, -np.inf, np.nan, np.finfo(np.float64).tiny], + dtype=np.float64, + ), + } + write_deterministic_npz(target / "elementwise.npz", elementwise) + + rng = make_rng(root_seed, "numerical_inputs", "indexing") + indexing = { + "source": rng.normal(size=(7, 11, 13)).astype(np.float64), + "row_indices": np.array([6, 0, 3, 3, 1], dtype=np.int64), + "column_indices": np.array([10, 2, 8, 1, 1], dtype=np.int64), + "gather_indices": rng.integers(0, 13, size=(7, 11, 5), dtype=np.int64), + "boolean_mask": (rng.random((7, 11, 13)) > 0.7), + "scatter_values": rng.normal(size=(7, 11, 5)).astype(np.float64), + } + write_deterministic_npz(target / "indexing.npz", indexing) + + rng = make_rng(root_seed, "numerical_inputs", "reductions") + cancellation_pattern = np.array([1e8, 1.0, -1e8, 3.0, -3.0], dtype=np.float64) + cancellation = np.resize(cancellation_pattern, reduction_rows * reduction_columns) + cancellation = cancellation.reshape(reduction_rows, reduction_columns) + reductions = { + "positive": rng.uniform( + 0.0, + 10.0, + size=(reduction_rows, reduction_columns), + ).astype(np.float64), + "mixed_sign": rng.normal( + size=(reduction_rows, reduction_columns), + ).astype(np.float64), + "cancellation": cancellation, + "cube": rng.normal(size=(5, 17, 23)).astype(np.float64), + "integer_values": rng.integers( + -100, + 101, + size=(reduction_rows, reduction_columns), + dtype=np.int64, + ), + } + write_deterministic_npz(target / "reductions.npz", reductions) + + rng = make_rng(root_seed, "numerical_inputs", "matrix_operations") + matrix_operations = { + "left": rng.normal(size=(matrix_m, matrix_k)).astype(np.float64), + "right": rng.normal(size=(matrix_k, matrix_n)).astype(np.float64), + "vector": rng.normal(size=(matrix_k,)).astype(np.float64), + "batch_left": rng.normal( + size=(matrix_batch_size, matrix_m, matrix_k), + ).astype(np.float64), + "batch_right": rng.normal( + size=(matrix_batch_size, matrix_k, matrix_n), + ).astype(np.float64), + "einsum_left": rng.normal(size=(5, 7, 11)).astype(np.float64), + "einsum_right": rng.normal(size=(11, 13)).astype(np.float64), + } + write_deterministic_npz(target / "matrix_operations.npz", matrix_operations) + + rng = make_rng(root_seed, "numerical_inputs", "convolutions") + convolutions = { + "conv1d_input": rng.normal(size=(2, 3, 31)).astype(np.float64), + "conv1d_weight": rng.normal(size=(4, 3, 5)).astype(np.float64), + "conv1d_bias": rng.normal(size=(4,)).astype(np.float64), + "conv2d_input": rng.normal(size=(2, 3, 17, 19)).astype(np.float64), + "conv2d_weight": rng.normal(size=(5, 3, 3, 3)).astype(np.float64), + "conv2d_bias": rng.normal(size=(5,)).astype(np.float64), + "grouped_conv2d_input": rng.normal(size=(2, 4, 16, 18)).astype(np.float64), + "grouped_conv2d_weight": rng.normal(size=(6, 2, 3, 3)).astype(np.float64), + "grouped_conv2d_bias": rng.normal(size=(6,)).astype(np.float64), + "conv3d_input": rng.normal(size=(1, 2, 9, 11, 13)).astype(np.float64), + "conv3d_weight": rng.normal(size=(3, 2, 3, 3, 3)).astype(np.float64), + "conv3d_bias": rng.normal(size=(3,)).astype(np.float64), + } + write_deterministic_npz(target / "convolutions.npz", convolutions) + + rng = make_rng(root_seed, "numerical_inputs", "linear_algebra") + dimension = 17 + orthogonal, _ = np.linalg.qr(rng.normal(size=(dimension, dimension))) + well_values = np.logspace(0.0, 2.0, dimension) + ill_values = np.logspace(0.0, 10.0, dimension) + well_conditioned = orthogonal @ np.diag(well_values) @ orthogonal.T + ill_conditioned = orthogonal @ np.diag(ill_values) @ orthogonal.T + spd = well_conditioned.T @ well_conditioned + np.eye(dimension) + eigenvalues = np.concatenate((np.array([1.0, 1.0, 1.0]), np.arange(2, dimension - 1))) + degenerate_symmetric = orthogonal @ np.diag(eigenvalues) @ orthogonal.T + linear_algebra = { + "well_conditioned_matrix": well_conditioned.astype(np.float64), + "well_conditioned_rhs": rng.normal(size=(dimension, 3)).astype(np.float64), + "ill_conditioned_matrix": ill_conditioned.astype(np.float64), + "ill_conditioned_rhs": rng.normal(size=(dimension, 2)).astype(np.float64), + "positive_definite_matrix": spd.astype(np.float64), + "rectangular_matrix": rng.normal(size=(23, 11)).astype(np.float64), + "svd_matrix": rng.normal(size=(19, 13)).astype(np.float64), + "degenerate_symmetric_matrix": degenerate_symmetric.astype(np.float64), + } + write_deterministic_npz(target / "linear_algebra.npz", linear_algebra) + + rng = make_rng(root_seed, "numerical_inputs", "fft") + fft_inputs = { + "real_1d": rng.normal(size=(257,)).astype(np.float64), + "real_2d": rng.normal(size=(31, 29)).astype(np.float64), + "complex_1d": ( + rng.normal(size=(257,)) + 1j * rng.normal(size=(257,)) + ).astype(np.complex128), + "complex_2d": ( + rng.normal(size=(17, 19)) + 1j * rng.normal(size=(17, 19)) + ).astype(np.complex128), + } + write_deterministic_npz(target / "fft.npz", fft_inputs) + + rng = make_rng(root_seed, "numerical_inputs", "special_functions") + special_functions = { + "positive": np.logspace(-6, 3, num=vector_length, dtype=np.float64), + "signed": rng.uniform(-8.0, 8.0, size=vector_length).astype(np.float64), + "probabilities": rng.uniform(1e-6, 1.0 - 1e-6, size=vector_length).astype( + np.float64 + ), + "gamma_inputs": rng.uniform(0.05, 20.0, size=vector_length).astype(np.float64), + "softmax_matrix": rng.normal(size=(31, 17)).astype(np.float64), + } + write_deterministic_npz(target / "special_functions.npz", special_functions) + + write_json( + target / "metadata.json", + { + "dataset_id": "numerical_inputs_v1", + "root_seed": root_seed, + "description": "Canonical inputs for core tensor and numerical kernel tests", + }, + ) + + +def generate_model_inputs( + target: Path, + root_seed: int, + config: Mapping[str, Any], + sms_config: Mapping[str, Any], +) -> None: + batch_size = require_int(config, "batch_size", minimum=2) + linear_input = require_int(config, "linear_input_features", minimum=2) + linear_output = require_int(config, "linear_output_features", minimum=2) + mlp_input = require_int(config, "mlp_input_features", minimum=2) + mlp_hidden = require_int_sequence(config, "mlp_hidden_features") + mlp_output = require_int(config, "mlp_output_features", minimum=2) + cnn_channels = require_int_sequence(config, "cnn_channels", expected_length=3) + cnn_classes = require_int(config, "cnn_classes", minimum=2) + attention_length = require_int(config, "attention_sequence_length", minimum=2) + embedding_size = require_int(config, "attention_embedding_size", minimum=4) + attention_heads = require_int(config, "attention_heads", minimum=1) + feedforward_size = require_int(config, "transformer_feedforward_size", minimum=4) + transformer_layers = require_int(config, "transformer_layers", minimum=1) + sms_sequence_length = require_int(sms_config, "max_sequence_length", minimum=4) + vocabulary_size = require_int(sms_config, "max_vocabulary_size", minimum=8) + + if embedding_size % attention_heads != 0: + raise DatasetGenerationError( + "attention_embedding_size must be divisible by attention_heads" + ) + + target.mkdir(parents=True, exist_ok=True) + rng = make_rng(root_seed, "model_inputs", "blocks") + block_inputs = { + "mlp_input": rng.normal(size=(batch_size, mlp_input)).astype(np.float32), + "mlp_labels": rng.integers(0, mlp_output, size=batch_size, dtype=np.int64), + "cnn_input": rng.normal(size=(batch_size, cnn_channels[0], 28, 28)).astype( + np.float32 + ), + "cnn_labels": rng.integers(0, cnn_classes, size=batch_size, dtype=np.int64), + "attention_input": rng.normal( + size=(batch_size, attention_length, embedding_size), + ).astype(np.float32), + "attention_padding_mask": ( + rng.random((batch_size, attention_length)) < 0.15 + ), + "attention_labels": rng.integers(0, 2, size=batch_size, dtype=np.int64), + } + block_inputs["attention_padding_mask"][:, 0] = False + write_deterministic_npz(target / "block_inputs.npz", block_inputs) + + rng = make_rng(root_seed, "model_inputs", "linear_state") + linear_model_state = linear_state( + rng, + "linear", + linear_input, + linear_output, + ) + write_deterministic_npz( + target / "linear_initial_state.npz", + linear_model_state, + ) + + rng = make_rng(root_seed, "model_inputs", "mlp_state") + mlp_state: dict[str, np.ndarray] = {} + layer_sizes = (mlp_input, *mlp_hidden, mlp_output) + for index, (input_size, output_size) in enumerate(zip(layer_sizes, layer_sizes[1:])): + mlp_state.update(linear_state(rng, f"layers.{index}", input_size, output_size)) + write_deterministic_npz(target / "mlp_initial_state.npz", mlp_state) + + rng = make_rng(root_seed, "model_inputs", "cnn_state") + cnn_state: dict[str, np.ndarray] = {} + cnn_state.update(conv2d_state(rng, "features.0", cnn_channels[0], cnn_channels[1], 3)) + cnn_state.update(conv2d_state(rng, "features.3", cnn_channels[1], cnn_channels[2], 3)) + flattened_features = cnn_channels[2] * 7 * 7 + cnn_state.update(linear_state(rng, "classifier.0", flattened_features, 64)) + cnn_state.update(linear_state(rng, "classifier.2", 64, cnn_classes)) + write_deterministic_npz(target / "cnn_initial_state.npz", cnn_state) + + rng = make_rng(root_seed, "model_inputs", "attention_state") + attention_state: dict[str, np.ndarray] = {} + attention_state.update(linear_state(rng, "q_proj", embedding_size, embedding_size)) + attention_state.update(linear_state(rng, "k_proj", embedding_size, embedding_size)) + attention_state.update(linear_state(rng, "v_proj", embedding_size, embedding_size)) + attention_state.update(linear_state(rng, "out_proj", embedding_size, embedding_size)) + attention_state["norm.weight"] = np.ones(embedding_size, dtype=np.float32) + attention_state["norm.bias"] = np.zeros(embedding_size, dtype=np.float32) + attention_state.update(linear_state(rng, "classifier", embedding_size, 2)) + write_deterministic_npz(target / "attention_initial_state.npz", attention_state) + + rng = make_rng(root_seed, "model_inputs", "sms_transformer_state") + transformer_state: dict[str, np.ndarray] = { + "token_embedding.weight": rng.normal( + 0.0, + embedding_size ** -0.5, + size=(vocabulary_size, embedding_size), + ).astype(np.float32), + "position_embedding.weight": rng.normal( + 0.0, + embedding_size ** -0.5, + size=(sms_sequence_length, embedding_size), + ).astype(np.float32), + } + transformer_state["token_embedding.weight"][0] = 0.0 + + for layer_index in range(transformer_layers): + prefix = f"encoder.layers.{layer_index}" + transformer_state[f"{prefix}.self_attn.in_proj_weight"] = xavier_uniform( + rng, + (3 * embedding_size, embedding_size), + embedding_size, + 3 * embedding_size, + ) + transformer_state[f"{prefix}.self_attn.in_proj_bias"] = np.zeros( + 3 * embedding_size, + dtype=np.float32, + ) + transformer_state.update( + linear_state( + rng, + f"{prefix}.self_attn.out_proj", + embedding_size, + embedding_size, + ) + ) + transformer_state.update( + linear_state(rng, f"{prefix}.linear1", embedding_size, feedforward_size) + ) + transformer_state.update( + linear_state(rng, f"{prefix}.linear2", feedforward_size, embedding_size) + ) + for norm_name in ("norm1", "norm2"): + transformer_state[f"{prefix}.{norm_name}.weight"] = np.ones( + embedding_size, + dtype=np.float32, + ) + transformer_state[f"{prefix}.{norm_name}.bias"] = np.zeros( + embedding_size, + dtype=np.float32, + ) + + transformer_state["final_norm.weight"] = np.ones(embedding_size, dtype=np.float32) + transformer_state["final_norm.bias"] = np.zeros(embedding_size, dtype=np.float32) + transformer_state.update(linear_state(rng, "classifier", embedding_size, 2)) + write_deterministic_npz( + target / "sms_transformer_initial_state.npz", + transformer_state, + ) + + write_json( + target / "metadata.json", + { + "dataset_id": "model_inputs_v1", + "root_seed": root_seed, + "attention_heads": attention_heads, + "description": "Fixed model inputs and initial states for block and workload tests", + }, + ) + + +def find_zip_member(archive: zipfile.ZipFile, expected_name: str) -> str: + matches = [name for name in archive.namelist() if Path(name).name == expected_name] + if len(matches) != 1: + raise DatasetGenerationError( + f"Expected one {expected_name!r} file in archive, found {len(matches)}" + ) + return matches[0] + + +def prepare_breast_cancer( + target: Path, + archive_path: Path, + root_seed: int, + config: Mapping[str, Any], +) -> None: + evaluation_fraction = require_float(config, "evaluation_fraction", 0.0, 1.0) + + with zipfile.ZipFile(archive_path) as archive: + member = find_zip_member(archive, "wdbc.data") + raw_text = archive.read(member).decode("utf-8") + + rows = list(csv.reader(io.StringIO(raw_text))) + if len(rows) != 569: + raise DatasetGenerationError(f"Expected 569 breast cancer rows, found {len(rows)}") + + features = np.empty((len(rows), 30), dtype=np.float64) + labels = np.empty(len(rows), dtype=np.int64) + identifiers = np.empty(len(rows), dtype=np.int64) + label_map = {"B": 0, "M": 1} + + for index, row in enumerate(rows): + if len(row) != 32: + raise DatasetGenerationError( + f"Breast cancer row {index} has {len(row)} columns instead of 32" + ) + identifiers[index] = int(row[0]) + try: + labels[index] = label_map[row[1]] + except KeyError as exc: + raise DatasetGenerationError(f"Unknown diagnosis label {row[1]!r}") from exc + features[index] = np.asarray(row[2:], dtype=np.float64) + + rng = make_rng(root_seed, "breast_cancer_wisconsin", "split") + train_indices, evaluation_indices = stratified_split_indices( + labels, + evaluation_fraction, + rng, + ) + + training_features = features[train_indices] + mean = training_features.mean(axis=0, dtype=np.float64) + standard_deviation = training_features.std(axis=0, dtype=np.float64) + if np.any(standard_deviation == 0.0): + raise DatasetGenerationError("A breast cancer feature has zero training variance") + + standardised = ((features - mean) / standard_deviation).astype(np.float32) + write_deterministic_npz( + target / "train.npz", + { + "features": standardised[train_indices], + "labels": labels[train_indices], + "source_indices": train_indices, + "source_identifiers": identifiers[train_indices], + }, + ) + write_deterministic_npz( + target / "evaluation.npz", + { + "features": standardised[evaluation_indices], + "labels": labels[evaluation_indices], + "source_indices": evaluation_indices, + "source_identifiers": identifiers[evaluation_indices], + }, + ) + write_deterministic_npz( + target / "preprocessing.npz", + { + "training_mean": mean, + "training_standard_deviation": standard_deviation, + }, + ) + write_json( + target / "metadata.json", + { + "dataset_id": "breast_cancer_wisconsin_v1", + "root_seed": root_seed, + "source_rows": len(rows), + "training_rows": len(train_indices), + "evaluation_rows": len(evaluation_indices), + "feature_count": features.shape[1], + "label_mapping": {"benign": 0, "malignant": 1}, + "evaluation_fraction": evaluation_fraction, + "standardisation": "Training split mean and population standard deviation", + }, + ) + + +def read_idx_images(path: Path) -> np.ndarray: + with gzip.open(path, "rb") as handle: + header = handle.read(16) + if len(header) != 16: + raise DatasetGenerationError(f"Invalid IDX image header in {path}") + magic, count, rows, columns = struct.unpack(">IIII", header) + if magic != 2051: + raise DatasetGenerationError(f"Unexpected IDX image magic {magic} in {path}") + raw = handle.read() + + expected_size = count * rows * columns + if len(raw) != expected_size: + raise DatasetGenerationError( + f"Expected {expected_size} image bytes in {path}, found {len(raw)}" + ) + return np.frombuffer(raw, dtype=np.uint8).reshape(count, rows, columns).copy() + + +def read_idx_labels(path: Path) -> np.ndarray: + with gzip.open(path, "rb") as handle: + header = handle.read(8) + if len(header) != 8: + raise DatasetGenerationError(f"Invalid IDX label header in {path}") + magic, count = struct.unpack(">II", header) + if magic != 2049: + raise DatasetGenerationError(f"Unexpected IDX label magic {magic} in {path}") + raw = handle.read() + + if len(raw) != count: + raise DatasetGenerationError( + f"Expected {count} label bytes in {path}, found {len(raw)}" + ) + return np.frombuffer(raw, dtype=np.uint8).copy() + + +def prepare_fashion_mnist( + target: Path, + source_paths: Sequence[Path], + root_seed: int, + config: Mapping[str, Any], +) -> None: + training_samples = require_int(config, "training_samples", minimum=10) + evaluation_samples = require_int(config, "evaluation_samples", minimum=10) + + train_images = read_idx_images(source_paths[0]) + train_labels = read_idx_labels(source_paths[1]) + evaluation_images = read_idx_images(source_paths[2]) + evaluation_labels = read_idx_labels(source_paths[3]) + + if train_images.shape != (60000, 28, 28) or train_labels.shape != (60000,): + raise DatasetGenerationError("Fashion-MNIST training files have unexpected shapes") + if evaluation_images.shape != (10000, 28, 28) or evaluation_labels.shape != (10000,): + raise DatasetGenerationError("Fashion-MNIST test files have unexpected shapes") + + training_indices = balanced_subset_indices( + train_labels, + training_samples, + make_rng(root_seed, "fashion_mnist", "training_subset"), + ) + evaluation_indices = balanced_subset_indices( + evaluation_labels, + evaluation_samples, + make_rng(root_seed, "fashion_mnist", "evaluation_subset"), + ) + + prepared_training = ( + train_images[training_indices, np.newaxis, :, :].astype(np.float32) / 255.0 + ) + prepared_evaluation = ( + evaluation_images[evaluation_indices, np.newaxis, :, :].astype(np.float32) / 255.0 + ) + + write_deterministic_npz( + target / "train.npz", + { + "images": prepared_training, + "labels": train_labels[training_indices].astype(np.int64), + "source_indices": training_indices, + }, + ) + write_deterministic_npz( + target / "evaluation.npz", + { + "images": prepared_evaluation, + "labels": evaluation_labels[evaluation_indices].astype(np.int64), + "source_indices": evaluation_indices, + }, + ) + write_json( + target / "metadata.json", + { + "dataset_id": "fashion_mnist_v1", + "root_seed": root_seed, + "training_rows": len(training_indices), + "evaluation_rows": len(evaluation_indices), + "image_shape": [1, 28, 28], + "classes": 10, + "normalisation": "uint8 pixel value divided by 255", + "augmentation": None, + }, + ) + + +def decode_text_file(raw: bytes) -> tuple[str, str]: + for encoding in ("utf-8", "utf-8-sig", "latin-1"): + try: + return raw.decode(encoding), encoding + except UnicodeDecodeError: + continue + raise DatasetGenerationError("Could not decode the SMS source file") + + +def tokenise_message(message: str) -> list[str]: + normalised = unicodedata.normalize("NFKC", message).casefold() + return TOKEN_PATTERN.findall(normalised) + + +def build_vocabulary( + messages: Sequence[str], + maximum_size: int, + minimum_frequency: int, +) -> dict[str, int]: + if maximum_size < len(SPECIAL_TOKENS): + raise DatasetGenerationError("max_vocabulary_size is smaller than the special tokens") + + counts: Counter[str] = Counter() + for message in messages: + counts.update(tokenise_message(message)) + + candidates = [ + (token, count) + for token, count in counts.items() + if count >= minimum_frequency and token not in SPECIAL_TOKENS + ] + candidates.sort(key=lambda item: (-item[1], item[0])) + kept = candidates[: maximum_size - len(SPECIAL_TOKENS)] + + vocabulary = {token: index for index, token in enumerate(SPECIAL_TOKENS)} + for token, _ in kept: + vocabulary[token] = len(vocabulary) + return vocabulary + + +def encode_messages( + messages: Sequence[str], + vocabulary: Mapping[str, int], + maximum_length: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + input_ids = np.zeros((len(messages), maximum_length), dtype=np.int64) + attention_mask = np.zeros((len(messages), maximum_length), dtype=np.bool_) + lengths = np.empty(len(messages), dtype=np.int64) + + unknown_id = vocabulary["[UNK]"] + beginning_id = vocabulary["[BOS]"] + end_id = vocabulary["[EOS]"] + + for row_index, message in enumerate(messages): + token_ids = [vocabulary.get(token, unknown_id) for token in tokenise_message(message)] + token_ids = [beginning_id, *token_ids[: maximum_length - 2], end_id] + length = len(token_ids) + input_ids[row_index, :length] = token_ids + attention_mask[row_index, :length] = True + lengths[row_index] = length + + return input_ids, attention_mask, lengths + + +def prepare_sms_spam( + target: Path, + archive_path: Path, + root_seed: int, + config: Mapping[str, Any], +) -> None: + evaluation_fraction = require_float(config, "evaluation_fraction", 0.0, 1.0) + maximum_length = require_int(config, "max_sequence_length", minimum=4) + maximum_vocabulary_size = require_int(config, "max_vocabulary_size", minimum=8) + minimum_frequency = require_int(config, "minimum_token_frequency", minimum=1) + + with zipfile.ZipFile(archive_path) as archive: + member = find_zip_member(archive, "SMSSpamCollection") + text, encoding = decode_text_file(archive.read(member)) + + messages: list[str] = [] + labels: list[int] = [] + label_map = {"ham": 0, "spam": 1} + for line_number, line in enumerate(text.splitlines(), start=1): + if not line: + continue + try: + raw_label, message = line.split("\t", 1) + label = label_map[raw_label] + except (ValueError, KeyError) as exc: + raise DatasetGenerationError(f"Invalid SMS row at line {line_number}") from exc + labels.append(label) + messages.append(message) + + label_array = np.asarray(labels, dtype=np.int64) + if len(messages) != 5574: + raise DatasetGenerationError(f"Expected 5574 SMS rows, found {len(messages)}") + + train_indices, evaluation_indices = stratified_split_indices( + label_array, + evaluation_fraction, + make_rng(root_seed, "sms_spam", "split"), + ) + training_messages = [messages[index] for index in train_indices] + evaluation_messages = [messages[index] for index in evaluation_indices] + vocabulary = build_vocabulary( + training_messages, + maximum_vocabulary_size, + minimum_frequency, + ) + + train_ids, train_mask, train_lengths = encode_messages( + training_messages, + vocabulary, + maximum_length, + ) + evaluation_ids, evaluation_mask, evaluation_lengths = encode_messages( + evaluation_messages, + vocabulary, + maximum_length, + ) + + write_deterministic_npz( + target / "train.npz", + { + "input_ids": train_ids, + "attention_mask": train_mask, + "lengths": train_lengths, + "labels": label_array[train_indices], + "source_indices": train_indices, + }, + ) + write_deterministic_npz( + target / "evaluation.npz", + { + "input_ids": evaluation_ids, + "attention_mask": evaluation_mask, + "lengths": evaluation_lengths, + "labels": label_array[evaluation_indices], + "source_indices": evaluation_indices, + }, + ) + write_json(target / "vocabulary.json", vocabulary) + write_json( + target / "metadata.json", + { + "dataset_id": "sms_spam_v1", + "root_seed": root_seed, + "source_rows": len(messages), + "training_rows": len(train_indices), + "evaluation_rows": len(evaluation_indices), + "vocabulary_size": len(vocabulary), + "configured_maximum_vocabulary_size": maximum_vocabulary_size, + "maximum_sequence_length": maximum_length, + "minimum_token_frequency": minimum_frequency, + "source_encoding": encoding, + "label_mapping": {"ham": 0, "spam": 1}, + "tokenisation": "Unicode NFKC, casefold, regex words and punctuation", + }, + ) + + +def collect_file_records(directory: Path) -> list[dict[str, Any]]: + if not directory.is_dir(): + return [] + records = [] + for path in sorted(candidate for candidate in directory.rglob("*") if candidate.is_file()): + records.append( + { + "relative_path": path.relative_to(DATASETS_DIR).as_posix(), + "sha256": hash_file(path, "sha256"), + "size_bytes": path.stat().st_size, + } + ) + return records + + +def update_manifest( + manifest: dict[str, Any], + suite_version: str, + root_seed: int, + selected_source_ids: set[str], +) -> None: + manifest["suite_name"] = "pytorch_extended_tests" + manifest["suite_version"] = suite_version + manifest["root_seed"] = root_seed + manifest["generated_at_utc"] = datetime.now(timezone.utc).isoformat() + + for source_id, source in manifest["sources"].items(): + for file_spec in source["files"]: + path = DATASETS_DIR / file_spec["relative_path"] + if not path.is_file(): + # Keep the recorded provenance when only generated inputs are refreshed + # The source archives do not need to stay in the repository after Level 6 preparation + if source_id in selected_source_ids: + file_spec["md5"] = None + file_spec["sha256"] = None + file_spec["size_bytes"] = None + continue + file_spec["md5"] = hash_file(path, "md5") + file_spec["sha256"] = hash_file(path, "sha256") + file_spec["size_bytes"] = path.stat().st_size + + for entry in manifest["generated_datasets"].values(): + entry["files"] = collect_file_records(DATASETS_DIR / entry["prepared_directory"]) + for entry in manifest["prepared_datasets"].values(): + entry["files"] = collect_file_records(DATASETS_DIR / entry["prepared_directory"]) + + write_json(MANIFEST_PATH, manifest) + + +def run_in_temporary_directory( + name: str, + target: Path, + force: bool, + action: Any, +) -> None: + PREPARED_DIR.mkdir(parents=True, exist_ok=True) + temporary_parent = Path(tempfile.mkdtemp(prefix=f".{name}-", dir=PREPARED_DIR)) + temporary_target = temporary_parent / name + temporary_target.mkdir() + try: + action(temporary_target) + replace_directory(temporary_target, target, force) + finally: + shutil.rmtree(temporary_parent, ignore_errors=True) + + +def load_manifest() -> dict[str, Any]: + try: + value = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise DatasetGenerationError(f"Could not read {MANIFEST_PATH}") from exc + if not isinstance(value, dict): + raise DatasetGenerationError("dataset_manifest.json must contain an object") + return value + + +def main() -> int: + args = parse_args() + suite_version, root_seed, generation_config = load_suite_configuration() + manifest = load_manifest() + + selected_source_ids: set[str] = set() + if args.only in ("all", "breast-cancer"): + selected_source_ids.add("breast_cancer_wisconsin_diagnostic") + if args.only in ("all", "fashion-mnist"): + selected_source_ids.add("fashion_mnist") + if args.only in ("all", "sms-spam"): + selected_source_ids.add("sms_spam_collection") + verify_manifest_sources(manifest, selected_source_ids) + + numerical_config = require_mapping(generation_config, "numerical_inputs") + model_config = require_mapping(generation_config, "model_inputs") + breast_config = require_mapping(generation_config, "breast_cancer_wisconsin") + fashion_config = require_mapping(generation_config, "fashion_mnist") + sms_config = require_mapping(generation_config, "sms_spam") + + if args.only in ("all", "generated"): + run_in_temporary_directory( + "numerical_inputs_v1", + PREPARED_DIR / "numerical_inputs_v1", + args.force, + lambda target: generate_numerical_inputs(target, root_seed, numerical_config), + ) + run_in_temporary_directory( + "model_inputs_v1", + PREPARED_DIR / "model_inputs_v1", + args.force, + lambda target: generate_model_inputs(target, root_seed, model_config, sms_config), + ) + + if args.only in ("all", "breast-cancer"): + breast_archive = source_file(manifest, "breast_cancer_wisconsin_diagnostic") + run_in_temporary_directory( + "breast_cancer_wisconsin_v1", + PREPARED_DIR / "breast_cancer_wisconsin_v1", + args.force, + lambda target: prepare_breast_cancer( + target, + breast_archive, + root_seed, + breast_config, + ), + ) + + if args.only in ("all", "fashion-mnist"): + fashion_paths = [ + source_file(manifest, "fashion_mnist", index) + for index in range(len(manifest["sources"]["fashion_mnist"]["files"])) + ] + run_in_temporary_directory( + "fashion_mnist_v1", + PREPARED_DIR / "fashion_mnist_v1", + args.force, + lambda target: prepare_fashion_mnist( + target, + fashion_paths, + root_seed, + fashion_config, + ), + ) + + if args.only in ("all", "sms-spam"): + sms_archive = source_file(manifest, "sms_spam_collection") + run_in_temporary_directory( + "sms_spam_v1", + PREPARED_DIR / "sms_spam_v1", + args.force, + lambda target: prepare_sms_spam( + target, + sms_archive, + root_seed, + sms_config, + ), + ) + + update_manifest(manifest, suite_version, root_seed, selected_source_ids) + print(f"Prepared datasets under {PREPARED_DIR}") + print(f"Updated {MANIFEST_PATH}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except DatasetGenerationError as exc: + print(f"Error: {exc}", file=sys.stderr) + raise SystemExit(2) from exc diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/evaluation.npz b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/evaluation.npz new file mode 100644 index 00000000..8b3fe21b Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/evaluation.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/metadata.json b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/metadata.json new file mode 100644 index 00000000..a93cb791 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/metadata.json @@ -0,0 +1,14 @@ +{ + "dataset_id": "breast_cancer_wisconsin_v1", + "evaluation_fraction": 0.2, + "evaluation_rows": 113, + "feature_count": 30, + "label_mapping": { + "benign": 0, + "malignant": 1 + }, + "root_seed": 42, + "source_rows": 569, + "standardisation": "Training split mean and population standard deviation", + "training_rows": 456 +} diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/preprocessing.npz b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/preprocessing.npz new file mode 100644 index 00000000..13dea647 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/preprocessing.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/train.npz b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/train.npz new file mode 100644 index 00000000..daadc508 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/breast_cancer_wisconsin_v1/train.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/evaluation.npz b/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/evaluation.npz new file mode 100644 index 00000000..574af013 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/evaluation.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/metadata.json b/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/metadata.json new file mode 100644 index 00000000..6e01639d --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/metadata.json @@ -0,0 +1,14 @@ +{ + "augmentation": null, + "classes": 10, + "dataset_id": "fashion_mnist_v1", + "evaluation_rows": 1024, + "image_shape": [ + 1, + 28, + 28 + ], + "normalisation": "uint8 pixel value divided by 255", + "root_seed": 42, + "training_rows": 4096 +} diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/train.npz b/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/train.npz new file mode 100644 index 00000000..54510115 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/fashion_mnist_v1/train.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/attention_initial_state.npz b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/attention_initial_state.npz new file mode 100644 index 00000000..7d7c22e4 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/attention_initial_state.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/block_inputs.npz b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/block_inputs.npz new file mode 100644 index 00000000..aa787d0c Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/block_inputs.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/cnn_initial_state.npz b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/cnn_initial_state.npz new file mode 100644 index 00000000..35809e44 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/cnn_initial_state.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/linear_initial_state.npz b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/linear_initial_state.npz new file mode 100644 index 00000000..6fef3f2c Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/linear_initial_state.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/metadata.json b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/metadata.json new file mode 100644 index 00000000..bb40aad7 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/metadata.json @@ -0,0 +1,6 @@ +{ + "attention_heads": 4, + "dataset_id": "model_inputs_v1", + "description": "Fixed model inputs and initial states for block and workload tests", + "root_seed": 42 +} diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/mlp_initial_state.npz b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/mlp_initial_state.npz new file mode 100644 index 00000000..5dd24b81 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/mlp_initial_state.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/sms_transformer_initial_state.npz b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/sms_transformer_initial_state.npz new file mode 100644 index 00000000..cded3140 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/model_inputs_v1/sms_transformer_initial_state.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/convolutions.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/convolutions.npz new file mode 100644 index 00000000..451f30b6 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/convolutions.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/elementwise.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/elementwise.npz new file mode 100644 index 00000000..399a8a67 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/elementwise.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/fft.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/fft.npz new file mode 100644 index 00000000..5193db0a Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/fft.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/indexing.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/indexing.npz new file mode 100644 index 00000000..81d310ba Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/indexing.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/linear_algebra.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/linear_algebra.npz new file mode 100644 index 00000000..404a9569 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/linear_algebra.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/matrix_operations.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/matrix_operations.npz new file mode 100644 index 00000000..df15c7c7 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/matrix_operations.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/metadata.json b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/metadata.json new file mode 100644 index 00000000..dc777535 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/metadata.json @@ -0,0 +1,5 @@ +{ + "dataset_id": "numerical_inputs_v1", + "description": "Canonical inputs for core tensor and numerical kernel tests", + "root_seed": 42 +} diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/reductions.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/reductions.npz new file mode 100644 index 00000000..5e59c4cf Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/reductions.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/special_functions.npz b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/special_functions.npz new file mode 100644 index 00000000..0cefbb11 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/numerical_inputs_v1/special_functions.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/evaluation.npz b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/evaluation.npz new file mode 100644 index 00000000..2b91e7fd Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/evaluation.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/metadata.json b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/metadata.json new file mode 100644 index 00000000..9f536791 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/metadata.json @@ -0,0 +1,17 @@ +{ + "configured_maximum_vocabulary_size": 4096, + "dataset_id": "sms_spam_v1", + "evaluation_rows": 1114, + "label_mapping": { + "ham": 0, + "spam": 1 + }, + "maximum_sequence_length": 64, + "minimum_token_frequency": 1, + "root_seed": 42, + "source_encoding": "utf-8", + "source_rows": 5574, + "tokenisation": "Unicode NFKC, casefold, regex words and punctuation", + "training_rows": 4460, + "vocabulary_size": 4096 +} diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/train.npz b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/train.npz new file mode 100644 index 00000000..e61f3ab0 Binary files /dev/null and b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/train.npz differ diff --git a/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/vocabulary.json b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/vocabulary.json new file mode 100644 index 00000000..d94ea1b6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/datasets/prepared/sms_spam_v1/vocabulary.json @@ -0,0 +1,4098 @@ +{ + "!": 11, + "\"": 57, + "#": 67, + "$": 526, + "%": 967, + "&": 16, + "'": 106, + "(": 134, + ")": 34, + "*": 53, + "+": 111, + ",": 8, + "-": 28, + ".": 4, + "/": 39, + "0": 1678, + "00": 1046, + "000": 408, + "000pes": 3817, + "008704050406": 2580, + "0089": 3818, + "0121": 3819, + "01223585236": 3820, + "01223585334": 2581, + "0125698789": 3821, + "02": 1047, + "0207": 2014, + "02072069400": 3822, + "02073162414": 2582, + "021": 2583, + "03": 832, + "04": 968, + "05": 1679, + "050703": 3823, + "0578": 2584, + "06": 1048, + "07": 2585, + "07008009200": 3824, + "07090201529": 3825, + "07090298926": 3826, + "07099833605": 3827, + "07123456789": 2586, + "0721072": 3828, + "07732584351": 3829, + "07734396839": 2587, + "07742676969": 3830, + "07753741225": 3831, + "0776xxxxxxx": 3832, + "07781482378": 2588, + "077xxx": 3833, + "078": 3834, + "07808": 3835, + "07808247860": 3836, + "07808726822": 3837, + "07815296484": 3838, + "07821230901": 2589, + "078498": 3839, + "0789xxxxxxx": 3840, + "0796xxxxxx": 3841, + "07973788240": 3842, + "07xxxxxxxxx": 2015, + "08": 3843, + "0800": 792, + "08000407165": 3844, + "08000776320": 2590, + "08000839402": 743, + "08000930705": 700, + "08000938767": 2591, + "08001950382": 2016, + "08002888812": 2592, + "08002986030": 2593, + "08002986906": 2594, + "08006344447": 2595, + "0808": 2017, + "0825": 2596, + "083": 3845, + "0844": 3846, + "08448350055": 3847, + "08448714184": 3848, + "0845": 2018, + "08450542832": 3849, + "08452810071": 3850, + "08452810073": 2597, + "08452810075over18's": 2598, + "0870": 1149, + "08700435505150p": 3851, + "08700469649": 3852, + "08700621170150p": 3853, + "08701213186": 3854, + "08701237397": 3855, + "08701417012": 2599, + "08701417012150p": 2600, + "087016248": 3856, + "08701752560": 3857, + "087018728737": 3858, + "0870241182716": 2601, + "08702490080": 3859, + "08702840625": 2019, + "08704050406": 3860, + "08704439680": 3861, + "08706091795": 3862, + "0870737910216yrs": 3863, + "08707500020": 3864, + "08707509020": 1473, + "08707808226": 3865, + "08708034412": 3866, + "08708800282": 3867, + "08709222922": 2602, + "0871": 2603, + "087104711148": 3868, + "08712101358": 2604, + "0871212025016": 3869, + "08712300220": 1474, + "08712317606": 2605, + "08712400200": 3870, + "08712400602450p": 2606, + "08712400603": 3871, + "08712402050": 2607, + "08712402578": 3872, + "08712402779": 3873, + "08712402902": 3874, + "08712404000": 3875, + "08712405020": 1680, + "08712405022": 2608, + "08712460324": 1150, + "0871277810710p": 3876, + "0871277810810": 3877, + "0871277810910p": 3878, + "087147123779am": 3879, + "08714712388": 3880, + "08714712394": 3881, + "08714712412": 3882, + "08714714011": 3883, + "08715203028": 3884, + "08715203649": 3885, + "08715203652": 3886, + "08715203677": 3887, + "08715203685": 3888, + "08715205273": 3889, + "08715500022": 3890, + "08715705022": 1681, + "08717111821": 3891, + "08717168528": 3892, + "08717205546": 3893, + "0871750": 2609, + "08717507382": 3894, + "08717509990": 3895, + "08717890890": 3896, + "08717895698": 3897, + "08717898035": 2610, + "08718711108": 3898, + "08718720201": 1475, + "08718723815": 3899, + "08718725756": 3900, + "08718726270": 3901, + "087187262701": 2020, + "08718726970": 3902, + "08718726971": 3903, + "08718726978": 3904, + "087187272008": 3905, + "08718727868": 3906, + "08718727870": 2021, + "08718727870150ppm": 3907, + "08718730555": 3908, + "08718730666": 2611, + "08718738001": 2612, + "08718738034": 3909, + "08719180219": 3910, + "08719180248": 3911, + "08719181259": 3912, + "08719181503": 3913, + "08719181513": 2613, + "08719839835": 3914, + "08719899217": 3915, + "08719899229": 3916, + "08719899230": 3917, + "09": 2022, + "09041940223": 3918, + "09050000301": 3919, + "09050000332": 3920, + "09050000460": 3921, + "09050000555": 3922, + "09050000878": 3923, + "09050000928": 3924, + "09050001808": 3925, + "09050002311": 3926, + "09050003091": 2614, + "09050090044": 1291, + "09050280520": 3927, + "09053750005": 3928, + "09056242159": 2615, + "09057039994": 3929, + "09058094454": 3930, + "09058094455": 3931, + "09058094507": 3932, + "09058094565": 2616, + "09058094583": 3933, + "09058094594": 3934, + "09058094597": 2617, + "09058094599": 2618, + "09058095201": 3935, + "09058097189": 3936, + "09058097218": 3937, + "09058098002": 3938, + "09058099801": 2619, + "09061104276": 3939, + "09061104283": 3940, + "09061209465": 2620, + "09061213237": 2621, + "09061221061": 3941, + "09061221066": 1682, + "09061701444": 3942, + "09061701461": 2622, + "09061701851": 3943, + "09061702893": 3944, + "09061743386": 2623, + "09061743806": 2624, + "09061743810": 3945, + "09061743811": 3946, + "09061749602": 3947, + "09061790121": 2625, + "09061790125": 3948, + "09061790126": 3949, + "09063440451": 3950, + "09063442151": 3951, + "09063458130": 2626, + "0906346330": 3952, + "09064011000": 2627, + "09064012160": 2628, + "09064017295": 3953, + "09064017305": 3954, + "09064018838": 3955, + "09064019014": 3956, + "09065069120": 3957, + "09065171142": 2629, + "09065174042": 2630, + "09065394514": 3958, + "09065394973": 3959, + "09065989182": 2631, + "09066350750": 2632, + "09066358152": 2633, + "09066362206": 3960, + "09066362220": 3961, + "09066362231": 2023, + "09066364311": 2634, + "09066364349": 3962, + "09066364589": 3963, + "09066368327": 3964, + "09066368470": 3965, + "09066368753": 3966, + "09066380611": 2635, + "09066382422": 2636, + "09066612661": 2637, + "09066649731from": 3967, + "09066660100": 3968, + "09071512433": 3969, + "09071517866": 3970, + "09077818151": 3971, + "09090204448": 3972, + "09094100151": 3973, + "09094646631": 3974, + "09094646899": 2638, + "09095350301": 3975, + "09096102316": 3976, + "09099725823": 3977, + "09099726395": 3978, + "09099726481": 3979, + "09099726553": 3980, + "09111030116": 3981, + "09111032124": 3982, + "09701213186": 3983, + "0a": 3984, + "1": 97, + "1's": 3985, + "10": 336, + "100": 326, + "100's": 3986, + "1000": 381, + "1000's": 3987, + "1000s": 2024, + "100p": 3988, + "100percent": 2025, + "1013": 3989, + "1030": 3990, + "10am": 1292, + "10k": 2026, + "10p": 527, + "10ppm": 2639, + "10th": 3991, + "11": 674, + "113": 3992, + "1131": 3993, + "114": 3994, + "1146": 3995, + "116": 3996, + "118p": 3997, + "11mths": 1476, + "11pm": 2027, + "12": 888, + "121": 2028, + "1225": 3998, + "123": 3999, + "125": 4000, + "1250": 4001, + "125gift": 2029, + "12hours": 4002, + "12hrs": 744, + "12mths": 4003, + "13": 2640, + "1327": 1293, + "14": 4004, + "140": 4005, + "1405": 4006, + "140ppm": 4007, + "145": 2030, + "1450": 2641, + "146tf150p": 4008, + "14tcr": 4009, + "14thmarch": 4010, + "15": 2031, + "150": 701, + "1500": 2642, + "150p": 211, + "150p16": 2643, + "150pm": 2032, + "150ppermesssubscription": 2033, + "150ppm": 382, + "150ppmpobox10183bhamb64xe": 4011, + "150ppmsg": 4012, + "150pw": 4013, + "153": 2644, + "15541": 4014, + "15pm": 4015, + "16": 285, + "165": 2645, + "1680": 4016, + "169": 2646, + "177": 2647, + "18": 278, + "180": 4017, + "1843": 4018, + "18p": 4019, + "18yrs": 2648, + "195": 4020, + "1956669": 2034, + "1apple": 4021, + "1cup": 4022, + "1da": 4023, + "1er": 4024, + "1hr": 2649, + "1im": 4025, + "1lemon": 4026, + "1million": 4027, + "1pm": 4028, + "1st": 422, + "1st4terms": 4029, + "1stchoice": 4030, + "1stone": 4031, + "1thing": 4032, + "1tulsi": 4033, + "1win150ppmx3": 2650, + "1winaweek": 2035, + "1winawk": 4034, + "1x150p": 1477, + "1yf": 4035, + "2": 29, + "20": 889, + "200": 890, + "2000": 599, + "2003": 891, + "2004": 1294, + "2005": 2036, + "2007": 2651, + "200p": 4036, + "2025050": 4037, + "20m12aq": 4038, + "20p": 1049, + "21": 2037, + "21870000": 2652, + "21st": 2038, + "22": 2653, + "220": 2654, + "220cm2": 4039, + "2309": 4040, + "23f": 4041, + "23g": 4042, + "24": 2039, + "24hrs": 2655, + "24m": 4043, + "24th": 4044, + "25": 1295, + "250": 675, + "250k": 2656, + "25p": 1296, + "26": 2040, + "2667": 4045, + "26th": 2657, + "27": 2658, + "28": 1683, + "2814032": 4046, + "28days": 4047, + "28th": 4048, + "28thfeb": 4049, + "29": 2041, + "2b": 4050, + "2bold": 4051, + "2c": 2659, + "2channel": 4052, + "2day": 1297, + "2docd": 4053, + "2end": 2660, + "2exit": 4054, + "2ez": 4055, + "2find": 2661, + "2getha": 4056, + "2geva": 2662, + "2go": 2042, + "2gthr": 4057, + "2hook": 4058, + "2hrs": 4059, + "2i": 4060, + "2lands": 833, + "2marrow": 2663, + "2moro": 1684, + "2morow": 2664, + "2morro": 4061, + "2morrow": 2043, + "2morrowxxxx": 4062, + "2mrw": 1478, + "2mwen": 4063, + "2nd": 466, + "2nights": 4064, + "2nite": 969, + "2optout": 1685, + "2p": 2665, + "2px": 4065, + "2rcv": 2666, + "2stop": 4066, + "2stoptxt": 2044, + "2u": 2667, + "2u2": 4067, + "2watershd": 4068, + "2waxsto": 4069, + "2wks": 2668, + "2wt": 2669, + "2years": 4070, + "2yr": 4071, + "3": 158, + "30": 745, + "300": 1479, + "3000": 4072, + "300603": 2670, + "3030": 1151, + "30ish": 2671, + "30pm": 4073, + "30pp": 4074, + "30s": 4075, + "30th": 4076, + "31": 2672, + "3100": 2673, + "310303": 4077, + "31p": 2674, + "32": 4078, + "3230": 4079, + "32323": 4080, + "326": 2675, + "33": 2045, + "350": 793, + "3510i": 1480, + "35p": 4081, + "3650": 2676, + "36504": 2046, + "3680": 2677, + "373": 2678, + "38": 4082, + "382": 4083, + "391784": 4084, + "3aj": 2679, + "3d": 1686, + "3days": 4085, + "3g": 1687, + "3gbp": 2680, + "3hrs": 4086, + "3lp": 2047, + "3mins": 1688, + "3mobile": 4087, + "3pound": 4088, + "3qxj9": 2048, + "3rd": 1689, + "3ss": 2681, + "3uz": 2682, + "3wks": 4089, + "3x": 4090, + "3xx": 4091, + "4": 54, + "4'o": 4092, + "40": 2683, + "400": 1152, + "400mins": 4093, + "400thousad": 4094, + "4041": 4095, + "40gb": 1690, + "41685": 2684, + "434": 1481, + "440": 1691, + "4403ldnw1a7rw18": 2685, + "447801259231": 2686, + "45239": 2687, + "4742": 2049, + "48": 2688, + "4eva": 2689, + "4fil": 2690, + "4get": 1692, + "4info": 2691, + "4mths": 2692, + "4t": 2050, + "4th": 1482, + "4u": 1298, + "5": 221, + "50": 217, + "500": 311, + "5000": 467, + "50gbp": 2051, + "50p": 1153, + "50perwksub": 2052, + "530": 1693, + "54": 2693, + "542": 1694, + "5k": 2694, + "5min": 2053, + "5p": 2054, + "5th": 2055, + "5wb": 1299, + "5we": 1483, + "6": 268, + "600": 2695, + "6031": 2696, + "60p": 1695, + "61610": 2697, + "62468": 1300, + "630": 2698, + "65": 2699, + "674": 2700, + "69696": 2701, + "69698": 2702, + "69888": 2703, + "6days": 2704, + "6hl": 2705, + "6hrs": 2056, + "6months": 2706, + "6th": 2057, + "7": 327, + "700": 2707, + "7250i": 2708, + "75": 2058, + "750": 639, + "762": 2709, + "77": 2710, + "786": 2059, + "7876150ppm": 2711, + "7pm": 1696, + "7th": 2060, + "8": 528, + "800": 600, + "80062": 1697, + "8007": 892, + "80082": 2712, + "80182": 2713, + "80488": 2061, + "80608": 2714, + "80878": 2062, + "81010": 2715, + "81151": 2063, + "81303": 2716, + "81618": 2717, + "82242": 2064, + "82277": 1301, + "83049": 2718, + "83222": 2719, + "83355": 2065, + "83383": 2720, + "83600": 2066, + "84025": 2721, + "84128": 2067, + "84199": 1698, + "85": 2722, + "85023": 1484, + "8552": 1699, + "86021": 2068, + "86688": 640, + "87021": 2723, + "87066": 834, + "87077": 1050, + "87121": 1700, + "87131": 1701, + "872": 2724, + "87239": 2725, + "87575": 2069, + "88039": 1702, + "88066": 2726, + "88088": 2727, + "88600": 2728, + "88877": 2729, + "88888": 2730, + "89070": 2731, + "89080": 2732, + "89545": 2070, + "89555": 2071, + "89693": 2733, + "8am": 2734, + "8p": 2735, + "8pm": 2736, + "8th": 1485, + "8wp": 2072, + "9": 449, + "900": 1051, + "910": 2737, + "9ae": 2073, + "9am": 1703, + "9ja": 2074, + "9pm": 2075, + "9t": 2738, + ":": 22, + ";": 20, + "<": 1486, + "=": 298, + ">": 394, + "?": 9, + "@": 337, + "[": 1704, + "[BOS]": 2, + "[EOS]": 3, + "[PAD]": 0, + "[UNK]": 1, + "\\": 2076, + "]": 1705, + "_": 2077, + "____": 2739, + "a": 10, + "aah": 2740, + "aathi": 1706, + "abi": 2741, + "abiola": 1302, + "abj": 2742, + "able": 548, + "about": 102, + "absolutly": 2743, + "abt": 676, + "abta": 2078, + "aburo": 2744, + "ac": 893, + "academic": 2745, + "acc": 1487, + "accept": 970, + "access": 1303, + "account": 363, + "ache": 1707, + "aco": 2746, + "across": 1154, + "acting": 2747, + "action": 1708, + "activate": 1488, + "active": 2748, + "activities": 2749, + "actor": 2750, + "actually": 395, + "ad": 2079, + "adam": 2751, + "add": 1489, + "added": 2752, + "addicted": 2753, + "addie": 2080, + "address": 549, + "admin": 2754, + "administrator": 2755, + "admirer": 894, + "adore": 2081, + "adoring": 2756, + "ads": 2757, + "adult": 2082, + "advance": 1709, + "advice": 1490, + "advise": 2758, + "affair": 2759, + "affairs": 2760, + "affection": 2083, + "afraid": 1710, + "aft": 702, + "after": 189, + "afternoon": 510, + "aftr": 2084, + "ag": 2761, + "again": 208, + "against": 1304, + "agalla": 2762, + "age": 835, + "age16": 971, + "ages": 2085, + "ago": 794, + "ah": 383, + "aha": 1305, + "ahead": 1306, + "ahmad": 1491, + "aight": 437, + "ain't": 2086, + "aint": 1492, + "air": 2087, + "airport": 1493, + "aiya": 2088, + "aiyah": 1307, + "aiyo": 1711, + "aka": 2763, + "al": 1155, + "alaipayuthe": 2764, + "album": 2765, + "alcohol": 2089, + "alert": 1712, + "alex": 1156, + "alfie": 2766, + "algarve": 2767, + "ali": 2768, + "all": 71, + "allah": 1713, + "allowed": 2769, + "almost": 836, + "alone": 795, + "along": 2090, + "already": 170, + "alright": 601, + "alrite": 1157, + "also": 198, + "always": 275, + "alwys": 2770, + "am": 83, + "amazing": 1494, + "american": 2771, + "among": 2772, + "amongst": 2773, + "amount": 2774, + "amp": 172, + "amt": 1714, + "an": 152, + "and": 14, + "andros": 2775, + "angry": 677, + "animation": 2776, + "annie": 2777, + "anniversary": 2778, + "announcement": 1308, + "anot": 2779, + "another": 384, + "ans": 1052, + "ansr": 1715, + "answer": 574, + "answered": 2780, + "answering": 2091, + "answers": 2092, + "anthony": 2781, + "anti": 2093, + "any": 127, + "anybody": 1716, + "anymore": 1158, + "anyone": 703, + "anything": 204, + "anytime": 704, + "anyway": 450, + "anyways": 2094, + "anywhere": 2782, + "aom": 2783, + "apartment": 1495, + "apo": 2784, + "apologise": 2095, + "app": 1309, + "apparently": 1717, + "application": 2785, + "apply": 438, + "appointment": 2096, + "appreciate": 1159, + "appreciated": 2786, + "apps": 2097, + "appt": 1718, + "april": 1310, + "ar": 1311, + "arcade": 1312, + "ard": 550, + "are": 32, + "area": 746, + "aren't": 1719, + "arent": 2787, + "argh": 2098, + "argue": 2099, + "argument": 2100, + "armand": 1720, + "arms": 2101, + "arng": 2788, + "around": 233, + "arrange": 1721, + "arrested": 1722, + "arrive": 1496, + "art": 2102, + "arts": 2789, + "arun": 2103, + "as": 89, + "asap": 972, + "ask": 187, + "askd": 973, + "asked": 529, + "askin": 1723, + "asking": 895, + "asks": 1497, + "asleep": 1313, + "asp": 2790, + "ass": 837, + "assume": 2104, + "at": 38, + "ate": 1498, + "atlanta": 1724, + "atm": 1725, + "attempt": 575, + "attend": 2105, + "auction": 796, + "audition": 2106, + "august": 2791, + "aunts": 2792, + "auto": 2107, + "available": 747, + "avatar": 2108, + "ave": 1053, + "avent": 2109, + "avoid": 2793, + "avoiding": 2794, + "await": 551, + "awaiting": 1160, + "awake": 1314, + "award": 491, + "awarded": 351, + "away": 468, + "awesome": 576, + "aww": 2795, + "ayn": 2796, + "b": 202, + "b'day": 974, + "b4": 838, + "b4190604": 2797, + "ba": 2798, + "babe": 193, + "babes": 1315, + "babies": 2799, + "baby": 511, + "back": 107, + "bad": 492, + "bag": 1316, + "bags": 2800, + "bak": 1054, + "balance": 896, + "bang": 2801, + "bank": 797, + "banks": 2110, + "bar": 2802, + "barely": 1499, + "base": 2803, + "basic": 2804, + "basically": 1726, + "bat": 2111, + "bath": 975, + "bathe": 976, + "bathing": 1727, + "battery": 1500, + "battle": 2805, + "bay": 2112, + "bb": 1501, + "bcm1896wc1n3xx": 2806, + "bcm4284": 2807, + "bcoz": 1161, + "bcums": 2113, + "bday": 1728, + "be": 50, + "beautiful": 602, + "bec": 2808, + "because": 364, + "become": 1317, + "becomes": 2809, + "becoz": 1729, + "bed": 493, + "bedrm": 2810, + "bedroom": 1162, + "been": 130, + "beer": 1730, + "befor": 2114, + "before": 212, + "beg": 2811, + "begin": 1502, + "behind": 1503, + "bein": 2115, + "being": 328, + "believe": 603, + "bell": 2812, + "belly": 1504, + "belovd": 1731, + "beloved": 2813, + "ben": 2814, + "beneficiary": 2815, + "best": 352, + "best1": 2116, + "bet": 2117, + "better": 396, + "between": 423, + "beware": 2118, + "beyond": 2119, + "bf": 1505, + "bhaji": 2816, + "bid": 1732, + "bids": 2817, + "big": 365, + "bigger": 2818, + "biggest": 2120, + "bill": 1055, + "billed": 2819, + "bills": 2820, + "bin": 1506, + "bird": 2821, + "birds": 2121, + "birla": 2122, + "birthdate": 2822, + "birthday": 338, + "bishan": 2123, + "bit": 317, + "bitch": 1733, + "bite": 2124, + "biz": 1318, + "bk": 2823, + "black": 1319, + "blackberry": 2824, + "blah": 2125, + "blake's": 2126, + "blame": 2127, + "blank": 1734, + "bleh": 2825, + "bless": 2826, + "blessed": 2827, + "blessings": 2128, + "blind": 2828, + "block": 2829, + "bloo": 2129, + "blood": 977, + "bloody": 1735, + "bloomberg": 2830, + "blow": 2831, + "blu": 2832, + "blue": 897, + "bluetooth": 1056, + "bmw": 2833, + "boat": 2834, + "body": 2835, + "bold": 1320, + "bone": 2836, + "bonus": 641, + "boo": 1507, + "book": 642, + "booked": 1508, + "booking": 2130, + "books": 2131, + "boost": 2132, + "booty": 2133, + "bootydelious": 2837, + "bored": 469, + "borin": 2838, + "boring": 1736, + "born": 1737, + "borrow": 2839, + "boss": 1057, + "boston": 2840, + "both": 552, + "bother": 1321, + "bottom": 2134, + "bought": 1322, + "bout": 705, + "bowl": 2135, + "box": 366, + "box326": 2841, + "box39822": 1738, + "box95qu": 2842, + "box97n7qp": 2136, + "boy": 470, + "boye": 2137, + "boys": 1323, + "boytoy": 748, + "brain": 2138, + "brand": 1509, + "brandy": 2843, + "break": 678, + "bright": 2139, + "brilliant": 2140, + "bring": 530, + "bringing": 2141, + "brings": 978, + "bristol": 2844, + "bro": 2142, + "broke": 1739, + "broken": 2845, + "bros": 2143, + "brothas": 2144, + "brother": 643, + "brought": 2145, + "bruv": 2846, + "bslvyl": 979, + "bstfrnd": 2146, + "bt": 397, + "btw": 1740, + "bucks": 1163, + "bud": 2147, + "budget": 1741, + "buff": 2847, + "buffet": 2848, + "bugis": 1742, + "build": 2849, + "bulbs": 2850, + "buns": 1510, + "burger": 2851, + "burning": 2852, + "burns": 2853, + "bus": 471, + "business": 2148, + "busy": 604, + "but": 35, + "buy": 227, + "buying": 1164, + "buzy": 2854, + "buzz": 1165, + "bx420": 1743, + "by": 93, + "bye": 2149, + "c": 153, + "c's": 605, + "c52": 2855, + "cabin": 1744, + "cafe": 1324, + "cake": 1166, + "cal": 1325, + "calculation": 2856, + "calicut": 2150, + "california": 2857, + "call": 26, + "call09050000327": 2858, + "call2optout": 839, + "callback": 2859, + "called": 409, + "caller": 898, + "callers": 2151, + "callertune": 1326, + "callin": 2152, + "calling": 577, + "calls": 439, + "cam": 2860, + "camcorder": 706, + "came": 512, + "camera": 341, + "campus": 1327, + "can": 41, + "can't": 249, + "canary": 2861, + "cancel": 1167, + "cancer": 1511, + "cann't": 2862, + "cannot": 1512, + "cant": 219, + "cappuccino": 2863, + "captain": 2153, + "car": 339, + "card": 749, + "cardiff": 2154, + "care": 213, + "cared": 2864, + "career": 2865, + "careful": 2155, + "carefully": 2156, + "cares": 2866, + "caring": 2867, + "carlos": 707, + "carry": 2868, + "cars": 1745, + "cartoon": 1513, + "case": 899, + "cash": 179, + "cashbin": 2869, + "castor": 2870, + "cat": 1746, + "catch": 1168, + "catching": 2157, + "caught": 2158, + "cause": 644, + "causing": 2871, + "cbe": 1747, + "cc": 1748, + "cd": 1169, + "cdgt": 2872, + "cds": 1749, + "celeb": 2159, + "celebrate": 2160, + "cell": 1328, + "center": 2161, + "centre": 1750, + "certainly": 2162, + "cha": 1751, + "chain": 2163, + "challenge": 2164, + "chance": 367, + "chances": 2873, + "change": 553, + "changed": 1329, + "changes": 2165, + "channel": 2874, + "character": 2166, + "charge": 606, + "charged": 1058, + "charges": 2167, + "charity": 1330, + "charles": 2875, + "chart": 2876, + "chasing": 2168, + "chat": 223, + "chatting": 2877, + "cheap": 900, + "cheaper": 1514, + "check": 385, + "checked": 1331, + "checking": 901, + "cheer": 2169, + "cheers": 1059, + "cheese": 2878, + "chennai": 902, + "cherish": 2879, + "chest": 2880, + "chicken": 2170, + "chikku": 708, + "child": 2171, + "childish": 1752, + "children": 1753, + "chill": 2172, + "chillin": 2173, + "china": 2174, + "chinese": 1515, + "chip": 2881, + "choice": 1516, + "choose": 709, + "chosen": 2175, + "christmas": 903, + "church": 2176, + "cine": 1170, + "cinema": 1332, + "citizen": 2177, + "city": 1754, + "claim": 155, + "claire": 2178, + "class": 302, + "clean": 1060, + "cleaning": 1755, + "clear": 2882, + "cleared": 2179, + "clearing": 2883, + "clearly": 2884, + "clever": 2885, + "click": 2180, + "clock": 2886, + "clos1": 2181, + "close": 645, + "closed": 1333, + "closer": 2182, + "club": 607, + "cm2": 2887, + "cme": 2888, + "cn": 2889, + "cnn": 2890, + "co": 279, + "coast": 2891, + "cock": 2183, + "code": 424, + "coffee": 1061, + "coin": 2892, + "coins": 1756, + "cold": 1334, + "colleagues": 980, + "collect": 451, + "collected": 2893, + "collecting": 2894, + "collection": 452, + "college": 750, + "colour": 554, + "com": 203, + "com1win150ppmx3age16": 1757, + "come": 76, + "comedy": 2184, + "comes": 531, + "comfort": 2895, + "comin": 1062, + "coming": 303, + "common": 2185, + "community": 2186, + "comp": 904, + "company": 751, + "compare": 2896, + "competition": 1517, + "complaint": 2897, + "complete": 1171, + "completed": 2898, + "completely": 1335, + "complimentary": 840, + "compromised": 2899, + "computer": 841, + "comuk": 1172, + "concentrate": 2900, + "concert": 2187, + "conditions": 1758, + "conducts": 2901, + "confidence": 1759, + "confirm": 981, + "confirmd": 2902, + "confirmed": 2188, + "conform": 2903, + "confused": 2904, + "congrats": 752, + "congratulations": 842, + "connect": 2189, + "connection": 1518, + "connections": 2905, + "considering": 1760, + "console": 2906, + "constant": 2907, + "constantly": 2190, + "contact": 234, + "contacted": 1519, + "content": 710, + "contents": 2908, + "continue": 2909, + "contract": 2910, + "control": 2911, + "convey": 982, + "convinced": 2912, + "cook": 2191, + "cooking": 2913, + "cool": 353, + "coping": 2914, + "copy": 1520, + "cornwall": 2192, + "correct": 983, + "cos": 199, + "cost": 329, + "costa": 1336, + "costs": 2193, + "could": 245, + "couldn't": 1761, + "count": 2915, + "countin": 2916, + "country": 1337, + "couple": 905, + "course": 906, + "cousin": 2917, + "cover": 1173, + "coz": 608, + "cr01327bt": 2918, + "cr9": 1338, + "crab": 2194, + "crack": 2195, + "cramps": 2196, + "crap": 2919, + "crash": 2920, + "crave": 907, + "crazy": 711, + "cream": 2197, + "created": 1521, + "credit": 908, + "credited": 2198, + "credits": 1063, + "creep": 2921, + "creepy": 2922, + "cricketer": 2923, + "crisis": 2924, + "crore": 2925, + "cross": 1762, + "croydon": 1339, + "cry": 1522, + "cs": 304, + "csh11": 2926, + "cud": 1523, + "cuddle": 2199, + "cum": 843, + "cup": 1064, + "curious": 2927, + "current": 1763, + "currently": 909, + "curry": 2928, + "cust": 1764, + "custcare": 910, + "custom": 2929, + "customer": 255, + "customers": 2930, + "cut": 984, + "cute": 1174, + "cutefrnd": 2200, + "cutting": 2931, + "cuz": 1175, + "cw25wx": 1765, + "d": 140, + "da": 116, + "dad": 513, + "daddy": 1340, + "dai": 2932, + "daily": 1524, + "damn": 1525, + "dance": 2933, + "dancing": 2934, + "dare": 2201, + "dark": 2202, + "darlin": 844, + "darling": 1526, + "darlings": 2935, + "darren": 845, + "dat": 368, + "date": 578, + "datebox1282essexcm61xn": 2936, + "dates": 2203, + "dating": 846, + "dave": 2204, + "day": 81, + "days": 342, + "dbuk": 2937, + "de": 532, + "dead": 1341, + "deal": 1065, + "dealing": 2938, + "dear": 131, + "dear1": 2205, + "dearer": 2206, + "dearly": 2939, + "death": 2207, + "december": 2208, + "decide": 985, + "decided": 753, + "deciding": 2940, + "decimal": 579, + "decision": 1766, + "decisions": 2941, + "dedicate": 2942, + "dedicated": 2943, + "deep": 1176, + "deepak": 2944, + "deeraj": 2945, + "def": 2209, + "definite": 2946, + "definitely": 1177, + "del": 986, + "deleted": 2947, + "delhi": 2948, + "deliver": 2210, + "delivered": 2949, + "deliveredtomorrow": 2211, + "delivery": 646, + "dem": 1767, + "demand": 2950, + "den": 354, + "denis": 2951, + "department": 2952, + "depends": 1527, + "depressed": 2953, + "derek": 2954, + "desert": 2955, + "desparate": 2956, + "desperate": 2957, + "despite": 2958, + "details": 580, + "detroit": 2959, + "deus": 2960, + "develop": 2961, + "devouring": 2962, + "dey": 1342, + "dhina": 2963, + "di": 2964, + "dick": 2212, + "dictionary": 2213, + "did": 135, + "did'nt": 1528, + "did't": 1768, + "didn": 1769, + "didn't": 318, + "didnt": 472, + "die": 911, + "died": 1343, + "diet": 2965, + "diff": 2214, + "differ": 2966, + "different": 987, + "difficult": 1066, + "dificult": 2215, + "digital": 1178, + "dignity": 2967, + "dime": 2968, + "din": 1179, + "ding": 2969, + "dining": 2970, + "dinner": 425, + "dint": 2216, + "direct": 988, + "directly": 1529, + "dirty": 2217, + "dis": 440, + "disclose": 2971, + "disconnect": 2972, + "discount": 989, + "discreet": 2973, + "discuss": 1770, + "discussed": 2974, + "dislikes": 2975, + "display": 2218, + "distance": 2219, + "distract": 2976, + "disturb": 1067, + "disturbing": 2977, + "division": 2978, + "diwali": 2979, + "dload": 2980, + "dnt": 1068, + "do": 42, + "doc": 2220, + "doctor": 1180, + "does": 398, + "doesn": 1771, + "doesn't": 847, + "doesnt": 1181, + "dog": 1344, + "dogging": 848, + "doggy": 2221, + "doin": 849, + "doing": 182, + "dokey": 2981, + "dollar": 2982, + "dollars": 1772, + "don": 453, + "don't": 117, + "donate": 2983, + "done": 312, + "dont": 109, + "door": 1182, + "doors": 2984, + "dot": 2985, + "double": 533, + "down": 250, + "download": 754, + "downloaded": 2986, + "downloads": 2222, + "dr": 2987, + "draw": 319, + "dream": 1069, + "dreams": 609, + "dress": 2223, + "dressed": 2988, + "dresser": 2989, + "drink": 712, + "drinkin": 2990, + "drinking": 2991, + "drinks": 1773, + "drive": 555, + "drivin": 2992, + "driving": 610, + "drms": 2993, + "drop": 647, + "dropped": 1774, + "drove": 2994, + "drpd": 2995, + "drug": 1530, + "drugs": 1070, + "drunk": 2224, + "dry": 1775, + "dubsack": 2225, + "duchess": 2996, + "dude": 494, + "due": 1345, + "dun": 261, + "dunno": 454, + "durban": 2997, + "during": 1183, + "dvd": 1346, + "e": 160, + "each": 679, + "earlier": 755, + "early": 473, + "earth": 1776, + "easier": 1777, + "east": 2998, + "eastenders": 2999, + "easter": 3000, + "easy": 410, + "eat": 411, + "eaten": 2226, + "eatin": 1531, + "eating": 1532, + "ebay": 3001, + "ec2a": 1533, + "edge": 3002, + "edison": 3003, + "ee": 3004, + "eerie": 1778, + "eg": 1071, + "egg": 2227, + "eggs": 3005, + "eh": 912, + "eight": 3006, + "either": 648, + "ela": 3007, + "elaine": 3008, + "electricity": 3009, + "else": 649, + "elsewhere": 3010, + "em": 1347, + "email": 756, + "embarassed": 3011, + "empty": 990, + "en": 3012, + "end": 386, + "ended": 1534, + "ending": 1072, + "ends": 850, + "enemy": 1779, + "energy": 1348, + "eng": 1535, + "england": 1073, + "english": 1780, + "enjoy": 369, + "enjoyed": 1781, + "enjoyin": 3013, + "enough": 534, + "enter": 798, + "entered": 991, + "entitled": 1536, + "entry": 514, + "entry41": 3014, + "enuff": 3015, + "envelope": 1537, + "er": 1538, + "ericsson": 3016, + "erm": 3017, + "error": 3018, + "escape": 3019, + "ese": 3020, + "especially": 1349, + "esplanade": 1782, + "essential": 2228, + "eta": 3021, + "etc": 992, + "euro2004": 1783, + "eurodisinc": 3022, + "europe": 3023, + "eve": 650, + "eveb": 3024, + "even": 235, + "evening": 474, + "evenings": 3025, + "ever": 387, + "every": 214, + "everybody's": 3026, + "everyone": 799, + "everything": 475, + "everywhere": 3027, + "evn": 3028, + "evng": 1539, + "evr": 3029, + "evrey": 3030, + "ex": 913, + "exact": 1784, + "exactly": 1785, + "exam": 993, + "exams": 1786, + "excellent": 1350, + "except": 1184, + "exciting": 2229, + "excuse": 1185, + "excuses": 3031, + "exe": 2230, + "executive": 3032, + "exeter": 3033, + "exhaust": 3034, + "exhausted": 3035, + "exmpel": 3036, + "expect": 1787, + "expecting": 1788, + "expensive": 1351, + "experience": 1186, + "expired": 3037, + "expires": 757, + "explain": 3038, + "explicit": 3039, + "explosive": 3040, + "express": 3041, + "expression": 3042, + "expressoffer": 3043, + "extra": 994, + "eye": 3044, + "eyes": 1187, + "f": 713, + "fa": 1789, + "face": 680, + "facebook": 1352, + "fact": 995, + "failed": 3045, + "fails": 3046, + "fair": 2231, + "faith": 3047, + "fake": 3048, + "fall": 1540, + "falls": 2232, + "family": 611, + "fancies": 2233, + "fancy": 758, + "fantasies": 1353, + "fantastic": 914, + "fantasy": 1790, + "far": 759, + "farm": 2234, + "fast": 714, + "faster": 1791, + "fat": 1541, + "father": 915, + "fathima": 3049, + "fault": 1074, + "fav": 3050, + "fave": 2235, + "favor": 3051, + "favorite": 3052, + "favour": 1792, + "favourite": 2236, + "fb": 1793, + "fear": 2237, + "feb": 1354, + "february": 1794, + "feel": 228, + "feelin": 2238, + "feeling": 651, + "feels": 996, + "fees": 2239, + "feet": 1795, + "fell": 2240, + "felt": 1075, + "female": 3053, + "fetch": 1542, + "few": 426, + "field": 3054, + "fifteen": 1796, + "fight": 1355, + "fighting": 1543, + "fightng": 2241, + "figure": 997, + "figures": 3055, + "file": 1797, + "files": 1798, + "fill": 2242, + "filled": 3056, + "filling": 3057, + "fills": 1799, + "film": 1188, + "films": 2243, + "filthy": 3058, + "final": 681, + "finally": 800, + "finance": 3059, + "find": 183, + "fine": 262, + "fingers": 1189, + "finish": 320, + "finished": 760, + "finishes": 2244, + "finishing": 3060, + "fire": 3061, + "first": 218, + "fish": 2245, + "fit": 2246, + "five": 1356, + "fix": 1800, + "fixed": 1190, + "fixedline": 3062, + "flag": 1076, + "flaked": 2247, + "flaky": 3063, + "flame": 3064, + "flash": 2248, + "flat": 2249, + "flies": 3065, + "flight": 2250, + "flights": 1801, + "flip": 3066, + "flirt": 1357, + "floor": 3067, + "flower": 1802, + "fly": 3068, + "fml": 3069, + "follow": 3070, + "followed": 1803, + "following": 1544, + "fone": 916, + "food": 612, + "fool": 2251, + "football": 3071, + "footprints": 2252, + "for": 21, + "force": 3072, + "forever": 917, + "forevr": 3073, + "forget": 851, + "forgets": 3074, + "forgot": 399, + "format": 3075, + "forms": 3076, + "forum": 3077, + "forums": 3078, + "forward": 1804, + "forwarded": 1077, + "found": 761, + "four": 3079, + "fr": 918, + "fran": 2253, + "frauds": 3080, + "freak": 3081, + "free": 66, + "free2day": 3082, + "freefone": 2254, + "freemsg": 919, + "freephone": 1191, + "freezing": 3083, + "fren": 1805, + "frens": 1358, + "fret": 3084, + "fri": 715, + "friday": 762, + "friend": 305, + "friend's": 2255, + "friends": 263, + "friendship": 556, + "fringe": 3085, + "frm": 763, + "frnd": 764, + "frnds": 765, + "frndship": 2256, + "frog": 3086, + "from": 65, + "fromm": 1806, + "frying": 2257, + "fuck": 495, + "fucked": 3087, + "fuckin": 2258, + "fucking": 766, + "ful": 2259, + "full": 652, + "fullonsms": 1545, + "fun": 535, + "funeral": 3088, + "funky": 3089, + "funny": 1192, + "furniture": 3090, + "further": 1807, + "future": 1193, + "fyi": 1808, + "g": 355, + "g696ga": 3091, + "gal": 998, + "galileo": 3092, + "gals": 1809, + "game": 557, + "games": 653, + "gamestar": 3093, + "ganesh": 3094, + "gang": 3095, + "gap": 1359, + "gaps": 1810, + "garage": 3096, + "garden": 3097, + "gardener": 3098, + "gary": 3099, + "gas": 920, + "gautham": 2260, + "gave": 1360, + "gay": 1078, + "gaytextbuddy": 3100, + "gbp": 3101, + "gd": 654, + "ge": 801, + "gee": 1361, + "geeee": 2261, + "geeeee": 3102, + "gender": 3103, + "generally": 2262, + "gent": 3104, + "gentle": 2263, + "gentleman": 3105, + "gently": 2264, + "genuine": 2265, + "george's": 3106, + "germany": 3107, + "get": 43, + "gets": 921, + "gettin": 999, + "getting": 280, + "getzed": 2266, + "gf": 3108, + "gibbs": 3109, + "gift": 496, + "gin": 3110, + "girl": 388, + "girlfrnd": 1362, + "girls": 716, + "gist": 3111, + "giv": 3112, + "give": 165, + "given": 2267, + "gives": 1363, + "giving": 1194, + "glad": 1079, + "gm": 1546, + "gn": 852, + "gnt": 3113, + "go": 64, + "go2": 3114, + "goals": 3115, + "god": 313, + "god's": 1195, + "gods": 3116, + "goes": 476, + "goin": 536, + "going": 92, + "gold": 3117, + "goldviking": 3118, + "gona": 2268, + "gone": 682, + "gonna": 251, + "good": 75, + "goodfriend": 3119, + "goodmorning": 717, + "goodnight": 1364, + "goodnite": 3120, + "goodnoon": 3121, + "goodo": 3122, + "google": 1547, + "gorgeous": 2269, + "gossip": 2270, + "got": 73, + "goto": 1196, + "gotta": 1080, + "govt": 1811, + "gpu": 3123, + "gr8": 802, + "gr8prizes": 3124, + "grace": 3125, + "grahmbell": 2271, + "gram": 2272, + "grand": 3126, + "granite": 3127, + "grave": 3128, + "gravity": 2273, + "great": 141, + "green": 1548, + "greet": 1549, + "greetings": 2274, + "grins": 1081, + "grl": 1812, + "ground": 2275, + "group": 1550, + "grow": 3129, + "gt": 59, + "guaranteed": 299, + "gud": 229, + "gudnite": 3130, + "guess": 441, + "guessing": 3131, + "guide": 2276, + "guilty": 2277, + "guy": 558, + "guys": 370, + "gving": 3132, + "gym": 1000, + "h": 1551, + "ha": 803, + "had": 175, + "haf": 581, + "haha": 291, + "hahaha": 2278, + "hai": 2279, + "hair": 497, + "haiz": 3133, + "half": 400, + "halloween": 2280, + "hamster": 3134, + "hand": 804, + "hands": 1813, + "handset": 1814, + "hanging": 2281, + "happen": 613, + "happend": 2282, + "happened": 767, + "happening": 1552, + "happens": 1365, + "happiness": 768, + "happy": 147, + "hard": 718, + "hardcore": 3135, + "hardly": 3136, + "harry": 1815, + "has": 148, + "hasn't": 1816, + "hate": 3137, + "hav": 537, + "have": 27, + "haven't": 614, + "havent": 559, + "havin": 2283, + "having": 330, + "havnt": 2284, + "he": 87, + "he'll": 1553, + "he's": 412, + "head": 719, + "headache": 3138, + "headin": 3139, + "heading": 3140, + "hear": 455, + "heard": 922, + "heart": 269, + "heater": 3141, + "heavy": 1366, + "hee": 1367, + "height": 2285, + "helen": 3142, + "hell": 1554, + "hella": 1555, + "hello": 292, + "help": 246, + "her": 129, + "here": 132, + "herself": 3143, + "hes": 3144, + "hey": 142, + "hg": 1082, + "hi": 114, + "hide": 3145, + "high": 1368, + "hill": 1817, + "him": 156, + "hint": 2286, + "hip": 2287, + "his": 236, + "history": 3146, + "hit": 805, + "hiya": 1556, + "hl": 1818, + "hlp": 3147, + "hmm": 806, + "hmmm": 807, + "hmv": 923, + "ho": 1083, + "hockey": 3148, + "hold": 808, + "holder": 1369, + "holding": 1370, + "holiday": 286, + "holla": 1557, + "hols": 2288, + "holy": 3149, + "home": 95, + "homeowners": 2289, + "hon": 2290, + "honey": 1197, + "honeybee": 2291, + "hook": 1558, + "hop": 1371, + "hope": 149, + "hoped": 3150, + "hopefully": 1819, + "hoping": 1198, + "hor": 3151, + "horny": 1372, + "horo": 3152, + "horrible": 2292, + "hospital": 1001, + "hostel": 1820, + "hot": 560, + "hotel": 1199, + "hour": 498, + "hours": 582, + "house": 331, + "housewives": 3153, + "how": 63, + "how's": 477, + "howard": 3154, + "however": 1559, + "hows": 809, + "howz": 1373, + "hppnss": 3155, + "hr": 1374, + "hrs": 1560, + "http": 615, + "hug": 1821, + "huge": 3156, + "huh": 683, + "hun": 2293, + "hundred": 3157, + "hungry": 810, + "hunny": 1375, + "hurry": 1822, + "hurt": 561, + "hurting": 3158, + "hurts": 1084, + "husband": 3159, + "hv": 2294, + "hw": 1823, + "i": 5, + "i'd": 924, + "i'll": 96, + "i'm": 44, + "i've": 247, + "iam": 1200, + "ias": 3160, + "ibh": 3161, + "ibhltd": 3162, + "ibiza": 1824, + "ibn": 3163, + "ice": 1201, + "icicibank": 3164, + "id": 616, + "idea": 1561, + "ideas": 3165, + "identifier": 811, + "idew": 3166, + "idiot": 2295, + "idk": 3167, + "if": 47, + "ignore": 1825, + "ikea": 3168, + "il": 1562, + "ill": 321, + "im": 184, + "imagine": 1376, + "imma": 1202, + "immediately": 1203, + "imp": 3169, + "important": 456, + "impossible": 2296, + "improve": 3170, + "improved": 3171, + "in": 17, + "in2": 3172, + "inc": 1377, + "inch": 3173, + "inches": 2297, + "incident": 2298, + "including": 1563, + "inclusive": 1826, + "indeed": 3174, + "india": 853, + "indian": 1827, + "indians": 2299, + "infections": 3175, + "infernal": 2300, + "info": 720, + "inform": 1828, + "information": 854, + "informed": 1829, + "infront": 3176, + "innings": 3177, + "insha": 2301, + "inside": 1204, + "installing": 3178, + "instantly": 3179, + "instead": 1830, + "instituitions": 1831, + "instructions": 3180, + "insurance": 1832, + "intelligent": 2302, + "interested": 1564, + "interesting": 1833, + "interflora": 3181, + "internet": 1205, + "interview": 2303, + "into": 371, + "intro": 2304, + "invaders": 3182, + "invest": 3183, + "invite": 3184, + "invited": 925, + "inviting": 1378, + "invnted": 2305, + "iouri": 2306, + "ip4": 1565, + "ipad": 3185, + "ipod": 1085, + "iq": 1834, + "irritates": 3186, + "irritating": 2307, + "is": 15, + "iscoming": 3187, + "ish": 1379, + "islands": 2308, + "isn": 3188, + "isn't": 1086, + "isnt": 1835, + "issue": 2309, + "issues": 1836, + "it": 24, + "it's": 171, + "italian": 2310, + "its": 77, + "itself": 1206, + "itz": 3189, + "ive": 1087, + "iz": 3190, + "izzit": 1207, + "j": 1380, + "jacket": 3191, + "jackpot": 3192, + "jada": 2311, + "james": 3193, + "jamster": 1837, + "jan": 2312, + "jane": 2313, + "january": 1208, + "jas": 3194, + "jason": 2314, + "java": 3195, + "jay": 1002, + "jay's": 3196, + "jazz": 1838, + "jealous": 3197, + "jen": 1839, + "jenny": 3198, + "jerry": 3199, + "jess": 2315, + "jesus": 2316, + "jiu": 1566, + "joanna": 3200, + "job": 343, + "jobs": 3201, + "jogging": 3202, + "john": 926, + "join": 478, + "joined": 1088, + "joining": 3203, + "joke": 1567, + "jokes": 2317, + "jokin": 3204, + "joking": 1840, + "jordan": 1568, + "journey": 3205, + "joy": 2318, + "joy's": 1381, + "jsco": 3206, + "jst": 1382, + "jstfrnd": 2319, + "jsut": 3207, + "juan": 3208, + "juicy": 2320, + "july": 2321, + "june": 3209, + "jus": 344, + "just": 51, + "juz": 562, + "k": 112, + "k52": 2322, + "kadeem": 2323, + "kaiez": 3210, + "kallis": 1209, + "kano": 1841, + "karaoke": 3211, + "kate": 1089, + "kb": 1569, + "ke": 3212, + "keep": 224, + "keeping": 1570, + "keeps": 3213, + "kegger": 3214, + "kept": 1383, + "kerala": 3215, + "kettoda": 2324, + "key": 2325, + "keys": 3216, + "kg": 3217, + "kick": 1384, + "kid": 3218, + "kids": 1003, + "kidz": 1842, + "killed": 3219, + "kills": 3220, + "kind": 721, + "kinda": 1210, + "king": 1385, + "kiosk": 3221, + "kiss": 457, + "kisses": 1843, + "kl341": 3222, + "knackered": 2326, + "knew": 1004, + "knock": 3223, + "know": 70, + "knowing": 2327, + "knows": 1005, + "knw": 722, + "konw": 3224, + "kudi": 3225, + "kusruthi": 2328, + "kz": 3226, + "l": 3227, + "l8r": 2329, + "l8tr": 3228, + "la": 1211, + "lab": 2330, + "lacs": 1844, + "ladies": 1845, + "lady": 1571, + "lag": 3229, + "laid": 1846, + "land": 723, + "landline": 401, + "landlines": 3230, + "lane": 3231, + "langport": 3232, + "laptop": 812, + "lar": 356, + "largest": 1847, + "last": 200, + "late": 237, + "lately": 3233, + "later": 124, + "latest": 427, + "latr": 1848, + "laugh": 1090, + "laughed": 2331, + "laughing": 1849, + "law": 1386, + "lays": 3234, + "lazy": 1091, + "lccltd": 3235, + "ldew": 1212, + "ldn": 1006, + "ldnw15h": 3236, + "le": 3237, + "lead": 1850, + "learn": 1387, + "least": 927, + "leave": 252, + "leaves": 1092, + "leaving": 855, + "lect": 1093, + "lecture": 3238, + "left": 413, + "legal": 1851, + "legs": 3239, + "leh": 442, + "lei": 563, + "lemme": 1388, + "leona": 2332, + "less": 1007, + "lesson": 684, + "lessons": 1213, + "let": 209, + "let's": 1389, + "lets": 617, + "letter": 2333, + "letters": 3240, + "lf56": 3241, + "liao": 357, + "library": 1390, + "lick": 3242, + "lido": 3243, + "lie": 1852, + "lies": 3244, + "life": 188, + "lifetime": 2334, + "lifpartnr": 2335, + "lift": 1214, + "light": 856, + "lik": 3245, + "like": 72, + "liked": 1215, + "likely": 2336, + "likes": 3246, + "lily": 3247, + "line": 372, + "linerental": 1853, + "lines": 2337, + "link": 1094, + "lion": 3248, + "list": 1572, + "listen": 1008, + "listening": 1854, + "literally": 3249, + "little": 402, + "live": 322, + "liverpool": 3250, + "lives": 2338, + "living": 1855, + "lk": 3251, + "ll": 389, + "lmao": 1391, + "lo": 3252, + "loads": 857, + "loan": 1009, + "loans": 3253, + "local": 1856, + "location": 3254, + "locations": 1573, + "lock": 3255, + "log": 724, + "login": 1574, + "logo": 1216, + "logos": 3256, + "lol": 222, + "london": 1392, + "lonely": 2339, + "long": 323, + "longer": 1393, + "look": 443, + "lookatme": 3257, + "looked": 1857, + "lookin": 3258, + "looking": 444, + "looks": 1575, + "loose": 3259, + "lor": 94, + "lose": 564, + "loses": 3260, + "losing": 2340, + "lost": 928, + "lot": 428, + "lots": 725, + "lou": 3261, + "loud": 2341, + "lounge": 3262, + "lousy": 3263, + "lov": 3264, + "lovable": 1217, + "love": 80, + "loved": 1010, + "lovely": 858, + "loveme": 3265, + "lover": 1218, + "loverboy": 1394, + "loves": 1219, + "loving": 685, + "lower": 3266, + "loxahatchee": 3267, + "loyal": 3268, + "loyalty": 1395, + "ls1": 3269, + "ls15hb": 3270, + "lst": 3271, + "lt": 60, + "ltd": 726, + "luck": 859, + "lucky": 686, + "lucozade": 3272, + "lucy": 3273, + "lunch": 332, + "lush": 3274, + "luv": 373, + "luxury": 2342, + "lvblefrnd": 2343, + "lyf": 3275, + "lyfu": 3276, + "m": 345, + "m227xy": 3277, + "m26": 3278, + "m263uz": 2344, + "m8": 2345, + "m8s": 3279, + "ma": 3280, + "maangalyam": 3281, + "mac": 3282, + "machan": 3283, + "macho": 3284, + "mad": 1858, + "made": 479, + "mag": 3285, + "maga": 3286, + "magical": 2346, + "mah": 929, + "maid": 1396, + "mail": 480, + "mailbox": 3287, + "mails": 3288, + "main": 3289, + "maintain": 3290, + "major": 3291, + "make": 151, + "makes": 618, + "making": 565, + "male": 3292, + "mall": 2347, + "man": 270, + "manage": 3293, + "managed": 3294, + "management": 2348, + "manda": 2349, + "maneesha": 2350, + "many": 256, + "map": 3295, + "march": 769, + "margaret": 2351, + "mark": 1095, + "market": 2352, + "marriage": 1397, + "married": 1398, + "marry": 2353, + "mas": 3296, + "masters": 3297, + "match": 860, + "matches": 1220, + "mate": 930, + "mates": 1221, + "math": 3298, + "matrix3": 3299, + "matter": 1576, + "matured": 2354, + "maturity": 3300, + "max": 3301, + "max10mins": 1399, + "maximize": 1577, + "may": 324, + "mayb": 770, + "maybe": 515, + "mca": 3302, + "mcat": 3303, + "me": 18, + "meal": 2355, + "mean": 583, + "meaning": 1578, + "means": 516, + "meant": 931, + "meanwhile": 1859, + "med": 2356, + "medical": 1860, + "medicine": 2357, + "meds": 1861, + "meet": 190, + "meetin": 2358, + "meeting": 306, + "meets": 3304, + "meh": 1096, + "mel": 3305, + "melle": 3306, + "member": 1579, + "members": 2359, + "membership": 2360, + "men": 1097, + "menu": 1222, + "meow": 3307, + "merry": 1098, + "mesages": 3308, + "message": 173, + "messaged": 3309, + "messages": 584, + "messaging": 3310, + "messenger": 3311, + "met": 932, + "mid": 1580, + "middle": 3312, + "midnight": 2361, + "mids": 3313, + "might": 390, + "miles": 2362, + "milk": 2363, + "min": 197, + "mind": 346, + "mine": 655, + "minmobsmorelkpobox177hp51fl": 3314, + "minmoremobsemspobox45po139wa": 3315, + "mins": 307, + "minute": 656, + "minutes": 414, + "minuts": 1862, + "miracle": 1099, + "mis": 3316, + "misbehaved": 3317, + "miss": 210, + "missed": 499, + "missin": 2364, + "missing": 429, + "mistake": 1581, + "mistakes": 3318, + "mite": 3319, + "mitsake": 3320, + "mix": 3321, + "mk45": 3322, + "mm": 1223, + "mmm": 2365, + "mmmm": 2366, + "mmmmm": 3323, + "mmmmmm": 3324, + "mnth": 3325, + "mo": 1224, + "moan": 1400, + "mob": 585, + "mobile": 125, + "mobiles": 861, + "mobilesdirect": 3326, + "mobileupd8": 727, + "mobno": 3327, + "moby": 1863, + "mode": 1100, + "model": 1225, + "module": 3328, + "moji": 2367, + "mojibiola": 3329, + "mokka": 3330, + "mom": 1011, + "mom's": 2368, + "moment": 1012, + "moments": 3331, + "moms": 1582, + "mon": 1101, + "monday": 933, + "money": 225, + "monkeys": 3332, + "mono": 3333, + "month": 415, + "month's": 3334, + "monthly": 3335, + "months": 1226, + "mood": 1227, + "moon": 1864, + "moon's": 3336, + "moral": 1102, + "more": 143, + "morefrmmob": 3337, + "morn": 2369, + "mornin": 3338, + "morning": 194, + "morphine": 2370, + "most": 481, + "mostly": 3339, + "mother": 1583, + "motorola": 934, + "mouth": 3340, + "move": 1013, + "moved": 3341, + "movie": 619, + "movies": 1103, + "movietrivia": 3342, + "mp3": 1584, + "mr": 935, + "mrng": 813, + "mrt": 1104, + "ms": 3343, + "msg": 144, + "msg150p": 3344, + "msging": 3345, + "msgrcvdhg": 2371, + "msgs": 771, + "msn": 3346, + "mt": 2372, + "mths": 1585, + "mtmsg18": 2373, + "mtmsgrcvd18": 2374, + "mu": 814, + "much": 161, + "mum": 772, + "mum's": 2375, + "mumtaz": 3347, + "mumtaz's": 3348, + "murder": 1865, + "murdered": 1401, + "murderer": 1402, + "music": 566, + "must": 517, + "musthu": 3349, + "muz": 1014, + "my": 19, + "myself": 1015, + "n": 110, + "n9dx": 3350, + "na": 862, + "nah": 1228, + "naked": 1229, + "nalla": 3351, + "name": 264, + "name1": 3352, + "name2": 3353, + "names": 3354, + "nap": 2376, + "nasdaq": 3355, + "nasty": 3356, + "nat": 3357, + "nat27081980": 3358, + "natalja": 3359, + "national": 586, + "natural": 2377, + "nature": 1866, + "naughty": 1230, + "nb": 3360, + "nd": 3361, + "ne": 2378, + "near": 1105, + "nearly": 1867, + "necessarily": 3362, + "necessary": 3363, + "necklace": 3364, + "ned": 3365, + "need": 91, + "needed": 1868, + "needs": 863, + "neft": 3366, + "neighbour": 3367, + "neither": 3368, + "nervous": 3369, + "net": 538, + "netcollex": 3370, + "network": 458, + "networking": 3371, + "networks": 2379, + "neva": 657, + "never": 281, + "new": 118, + "neway": 3372, + "newest": 2380, + "news": 658, + "next": 300, + "ni8": 864, + "nice": 253, + "nichols": 3373, + "nigeria": 1231, + "night": 145, + "nights": 1869, + "nimya": 1586, + "nit": 3374, + "nite": 567, + "nitros": 3375, + "no": 48, + "no1": 1870, + "nobody": 1587, + "noe": 687, + "nokia": 238, + "nokia6600": 3376, + "nokias": 3377, + "noline": 3378, + "none": 3379, + "noon": 1016, + "nope": 865, + "norm150p": 2381, + "normal": 1403, + "normally": 2382, + "northampton": 3380, + "nos": 3381, + "not": 36, + "note": 1871, + "nothin": 3382, + "nothing": 374, + "notice": 1404, + "notxt": 3383, + "noun": 3384, + "now": 33, + "nt": 815, + "ntt": 1017, + "ntwk": 1872, + "num": 1588, + "number": 166, + "numbers": 866, + "nuther": 3385, + "nvm": 1873, + "nw": 2383, + "nxt": 1589, + "nyc": 3386, + "nydc": 3387, + "nyt": 816, + "o": 568, + "o2": 1405, + "obviously": 1874, + "odi": 3388, + "of": 25, + "off": 248, + "offer": 430, + "offers": 936, + "office": 459, + "official": 1406, + "officially": 3389, + "ofice": 3390, + "often": 1875, + "oh": 138, + "oi": 3391, + "oic": 3392, + "oil": 3393, + "ok": 62, + "okay": 500, + "okey": 2384, + "okie": 620, + "ola": 3394, + "old": 587, + "omg": 1232, + "omw": 1233, + "on": 31, + "once": 431, + "one": 99, + "ones": 1234, + "oni": 1876, + "onion": 3395, + "online": 773, + "only": 84, + "onto": 1235, + "onwards": 2385, + "oooh": 3396, + "oops": 1106, + "open": 621, + "opening": 3397, + "operator": 1018, + "opinion": 1590, + "opportunity": 2386, + "opt": 728, + "option": 3398, + "optout": 1591, + "or": 40, + "or2stoptxt": 3399, + "orange": 445, + "orchard": 937, + "order": 688, + "ordered": 3400, + "oredi": 867, + "oreo": 3401, + "orig": 3402, + "original": 2387, + "oru": 3403, + "os": 3404, + "oso": 622, + "other": 239, + "others": 1592, + "otherwise": 1107, + "our": 115, + "out": 69, + "outside": 817, + "outta": 1877, + "over": 215, + "ow": 3405, + "own": 1108, + "owns": 3406, + "oz": 1878, + "p": 868, + "pa": 416, + "pack": 1879, + "package": 1880, + "page": 1593, + "pages": 2388, + "paid": 1594, + "pain": 501, + "painful": 3407, + "painting": 3408, + "pan": 3409, + "panic": 3410, + "paper": 938, + "papers": 3411, + "paperwork": 3412, + "parco": 3413, + "parent": 3414, + "parents": 729, + "paris": 3415, + "park": 939, + "parked": 3416, + "parking": 3417, + "part": 482, + "partner": 1595, + "partnership": 3418, + "party": 774, + "pass": 1236, + "passed": 3419, + "passionate": 3420, + "password": 1407, + "passwords": 3421, + "past": 1596, + "pattern": 3422, + "pay": 518, + "payee": 3423, + "paying": 1237, + "payment": 3424, + "payoh": 2389, + "pc": 1109, + "peace": 2390, + "peak": 3425, + "pee": 3426, + "pen": 3427, + "pence": 3428, + "pending": 2391, + "people": 293, + "per": 257, + "perfect": 1881, + "perhaps": 3429, + "period": 2392, + "person": 294, + "personal": 1882, + "personality": 2393, + "persons": 1408, + "pete": 940, + "petrol": 2394, + "pg": 1883, + "ph": 2395, + "philosophy": 3430, + "phne": 2396, + "phoenix": 3431, + "phone": 122, + "phoned": 3432, + "phones": 818, + "photo": 1884, + "photos": 1885, + "pic": 623, + "pick": 220, + "picked": 1238, + "picking": 869, + "pickle": 3433, + "pics": 689, + "picsfree1": 3434, + "picture": 2397, + "pictures": 1886, + "pie": 3435, + "piece": 3436, + "pieces": 2398, + "pig": 1887, + "pilates": 1597, + "pin": 1409, + "pink": 3437, + "piss": 3438, + "pissed": 1598, + "pix": 1239, + "pizza": 1410, + "place": 258, + "placement": 3439, + "places": 2399, + "plan": 375, + "plane": 3440, + "planet": 3441, + "planned": 1411, + "planning": 1110, + "plans": 941, + "play": 391, + "played": 1888, + "player": 775, + "players": 1412, + "playing": 2400, + "plaza": 3442, + "please": 126, + "pleased": 1599, + "pleasure": 1240, + "plenty": 1413, + "plm": 3443, + "pls": 123, + "plus": 502, + "plz": 539, + "pm": 588, + "po": 432, + "pobox": 942, + "pobox334": 1414, + "pobox36504w45wq": 1889, + "pobox45w2tg150p": 3444, + "pobox84": 1890, + "pocketbabe": 2401, + "pod": 2402, + "poem": 3445, + "point": 776, + "points": 659, + "pole": 3446, + "police": 1415, + "politicians": 3447, + "polo": 3448, + "poly": 730, + "polyph": 3449, + "polyphonic": 2403, + "polys": 1416, + "pongal": 1891, + "pool": 3450, + "poop": 3451, + "poor": 1417, + "pop": 2404, + "popcorn": 3452, + "popped": 2405, + "porn": 1892, + "position": 3453, + "possession": 3454, + "possible": 1418, + "post": 624, + "postcode": 2406, + "posted": 1893, + "potato": 3455, + "potential": 2407, + "potter": 3456, + "pouch": 3457, + "pound": 1111, + "pounds": 660, + "pours": 3458, + "pouts": 2408, + "power": 1241, + "ppl": 1600, + "pple": 1894, + "ppm": 2409, + "prabha": 1601, + "practical": 3459, + "practice": 2410, + "practicing": 3460, + "pray": 1419, + "praying": 3461, + "pre": 3462, + "prefer": 3463, + "preferably": 2411, + "premier": 3464, + "premium": 3465, + "prepare": 1895, + "prescription": 3466, + "present": 1420, + "press": 1112, + "pretty": 943, + "previous": 2412, + "previously": 2413, + "prey": 2414, + "price": 589, + "pride": 3467, + "prince": 3468, + "princess": 446, + "print": 3469, + "private": 690, + "prize": 180, + "prob": 777, + "probably": 483, + "problem": 403, + "problems": 1421, + "probs": 3470, + "process": 1602, + "prof": 3471, + "profit": 3472, + "program": 3473, + "project": 1113, + "prolly": 2415, + "promise": 1422, + "promises": 3474, + "promo": 3475, + "prompts": 3476, + "properly": 3477, + "propose": 3478, + "prospects": 2416, + "protect": 3479, + "prove": 2417, + "provided": 2418, + "ps": 3480, + "ptbo": 2419, + "pub": 778, + "public": 1896, + "purchase": 2420, + "purity": 2421, + "purpose": 1897, + "push": 3481, + "pushes": 3482, + "pussy": 3483, + "put": 484, + "putting": 1423, + "q": 1424, + "qatar": 1603, + "quality": 1604, + "queen": 1898, + "question": 590, + "questioned": 3484, + "questions": 819, + "quick": 1242, + "quickly": 2422, + "quiet": 3485, + "quit": 1899, + "quite": 404, + "quiz": 779, + "quote": 1605, + "quoting": 1606, + "r": 108, + "racing": 3486, + "radio": 3487, + "raed": 3488, + "rael": 3489, + "rain": 1019, + "raining": 1425, + "raise": 3490, + "raj": 3491, + "raji": 2423, + "rakhesh": 1114, + "rally": 3492, + "ran": 2424, + "random": 1426, + "randomly": 2425, + "rang": 3493, + "ranjith": 2426, + "rate": 433, + "rates": 1115, + "rather": 1427, + "rays": 1900, + "rcvd": 1116, + "rd": 1901, + "re": 540, + "reach": 434, + "reached": 1117, + "reaching": 1118, + "reaction": 3494, + "read": 820, + "readers": 3495, + "reading": 870, + "ready": 308, + "real": 333, + "realise": 3496, + "reality": 3497, + "realize": 2427, + "really": 185, + "realy": 1428, + "reason": 871, + "reasonable": 3498, + "reasons": 3499, + "rec": 3500, + "recd": 1902, + "receipt": 2428, + "receipts": 3501, + "receive": 358, + "receivea": 3502, + "received": 1903, + "receiving": 1904, + "recent": 3503, + "recently": 1243, + "recharge": 3504, + "reckon": 3505, + "record": 3506, + "records": 2429, + "recovery": 3507, + "red": 1119, + "redeemed": 1020, + "ref": 2430, + "reference": 1429, + "refused": 1905, + "reg": 3508, + "regarding": 2431, + "regards": 1430, + "register": 2432, + "registered": 1244, + "regret": 2433, + "regular": 2434, + "relation": 2435, + "relax": 2436, + "released": 2437, + "rem": 2438, + "remain": 2439, + "remains": 3509, + "remember": 405, + "remembered": 2440, + "remembr": 3510, + "remind": 1906, + "reminder": 2441, + "reminding": 3511, + "remove": 1245, + "removed": 3512, + "rent": 1120, + "rental": 1021, + "rentl": 3513, + "rents": 3514, + "repair": 3515, + "replied": 2442, + "reply": 113, + "replying": 1246, + "report": 1607, + "representative": 1247, + "request": 1608, + "requests": 2443, + "research": 1907, + "respect": 1908, + "respectful": 3516, + "responce": 3517, + "respond": 1909, + "responding": 3518, + "response": 3519, + "rest": 1609, + "restaurant": 3520, + "result": 2444, + "results": 2445, + "resume": 3521, + "retrieve": 3522, + "return": 1910, + "returned": 3523, + "returns": 1610, + "reveal": 821, + "review": 1911, + "revision": 3524, + "reward": 944, + "rewarding": 3525, + "rhythm": 3526, + "rice": 3527, + "rich": 2446, + "ride": 1912, + "right": 181, + "rightly": 3528, + "rights": 3529, + "ring": 661, + "ringtone": 503, + "ringtoneking": 2447, + "ringtones": 1248, + "rite": 625, + "river": 3530, + "road": 1249, + "roads": 3531, + "roast": 2448, + "rock": 945, + "rocks": 3532, + "rofl": 2449, + "roger": 1913, + "role": 1914, + "romantic": 2450, + "ron": 3533, + "room": 392, + "roommate": 2451, + "roommates": 3534, + "rooms": 1915, + "rose": 1611, + "round": 1250, + "row": 731, + "rply": 1022, + "rs": 946, + "rstm": 3535, + "ru": 2452, + "rub": 3536, + "rude": 1916, + "ruin": 3537, + "ruining": 3538, + "rule": 3539, + "rum": 3540, + "run": 626, + "running": 2453, + "runs": 2454, + "rush": 1917, + "s": 176, + "sachin": 2455, + "sacrifice": 3541, + "sad": 627, + "sae": 541, + "safe": 1121, + "said": 174, + "sake": 3542, + "salam": 3543, + "salary": 2456, + "sale": 1431, + "sales": 3544, + "salon": 3545, + "sam": 1612, + "same": 282, + "santa": 1918, + "sar": 3546, + "sarasota": 3547, + "sary": 3548, + "sat": 485, + "sathya": 3549, + "satisfied": 2457, + "satisfy": 3550, + "saturday": 780, + "saucy": 3551, + "savamob": 1122, + "save": 1023, + "saved": 2458, + "saw": 569, + "say": 177, + "saying": 662, + "says": 504, + "scared": 3552, + "scary": 2459, + "sch": 519, + "schedule": 2460, + "school": 520, + "schools": 3553, + "score": 2461, + "scoring": 3554, + "scotch": 3555, + "scotland": 3556, + "scotsman": 3557, + "scream": 1432, + "screaming": 1613, + "screen": 3558, + "scrounge": 3559, + "sd": 3560, + "sea": 822, + "search": 691, + "searching": 1919, + "season": 1614, + "seat": 2462, + "sec": 1920, + "second": 823, + "seconds": 3561, + "secret": 732, + "secretly": 3562, + "secs": 3563, + "sed": 1615, + "see": 105, + "seeds": 3564, + "seeing": 1024, + "seem": 2463, + "seemed": 1921, + "seems": 1251, + "seen": 1025, + "selected": 460, + "selection": 1616, + "self": 1433, + "selfish": 3565, + "sell": 1123, + "selling": 1922, + "sem": 3566, + "semester": 1026, + "sen": 1617, + "send": 85, + "sender": 2464, + "sending": 1252, + "sends": 3567, + "sense": 1618, + "sensitive": 3568, + "sent": 205, + "sept": 1923, + "series": 3569, + "serious": 1619, + "seriously": 1253, + "service": 283, + "services": 781, + "serving": 3570, + "set": 663, + "setting": 3571, + "settings": 1924, + "settle": 2465, + "settled": 2466, + "seven": 3572, + "several": 3573, + "sex": 872, + "sexy": 447, + "sh": 3574, + "sha": 1925, + "shagged": 3575, + "shall": 435, + "shame": 1926, + "shampain": 3576, + "share": 1254, + "sharing": 3577, + "shd": 1434, + "she": 119, + "she'll": 1927, + "she's": 692, + "sheets": 2467, + "shesil": 3578, + "shijas": 3579, + "ship": 3580, + "shipped": 3581, + "shipping": 2468, + "shirt": 2469, + "shirts": 2470, + "shit": 359, + "shld": 2471, + "shoes": 3582, + "shoot": 2472, + "shop": 570, + "shoppin": 3583, + "shopping": 461, + "shore": 3584, + "short": 1435, + "shortage": 3585, + "shorter": 2473, + "shortly": 1928, + "shot": 3586, + "should": 167, + "shouldn't": 2474, + "show": 542, + "shower": 947, + "showing": 2475, + "shows": 417, + "shracomorsglsuplt": 3587, + "shu": 3588, + "shuhui": 1436, + "shut": 3589, + "shy": 3590, + "si": 1929, + "sib": 3591, + "sick": 1124, + "side": 1027, + "sigh": 3592, + "sighs": 1620, + "sight": 1621, + "sign": 1255, + "silence": 3593, + "silent": 1437, + "silently": 3594, + "silver": 3595, + "sim": 1622, + "simple": 628, + "simpler": 3596, + "simply": 1623, + "since": 664, + "sinco": 3597, + "sing": 1930, + "single": 1438, + "singles": 1931, + "sipix": 1624, + "sir": 360, + "sis": 824, + "sister": 733, + "sit": 1625, + "site": 1932, + "sitll": 3598, + "sitting": 1256, + "situation": 1439, + "siva": 3599, + "six": 1933, + "size": 3600, + "sk3": 2476, + "sk38xh": 1257, + "skilgme": 1934, + "skillgame": 2477, + "skip": 3601, + "sky": 1258, + "skype": 3602, + "skyped": 3603, + "slap": 2478, + "slave": 1125, + "sleep": 254, + "sleepin": 3604, + "sleeping": 591, + "sleepy": 3605, + "slept": 1259, + "slightly": 3606, + "slippers": 3607, + "slo": 3608, + "slots": 3609, + "slow": 1126, + "slowly": 1127, + "small": 948, + "smart": 1935, + "smashed": 3610, + "smile": 334, + "smiles": 1936, + "smiling": 665, + "smoke": 734, + "smokes": 2479, + "smoking": 2480, + "sms": 271, + "smsco": 3611, + "smth": 735, + "sn": 1937, + "snake": 2481, + "snow": 782, + "snowman": 3612, + "so": 37, + "social": 1938, + "sofa": 1028, + "soft": 2482, + "software": 3613, + "sol": 1440, + "solve": 2483, + "some": 137, + "some1": 1626, + "somebody": 1029, + "someone": 240, + "somethin": 1627, + "something": 230, + "sometime": 3614, + "sometimes": 1441, + "somewhere": 1128, + "song": 873, + "songs": 1939, + "sony": 1030, + "sonyericsson": 1940, + "soo": 3615, + "soon": 265, + "sooner": 1941, + "sore": 1942, + "sorrow": 3616, + "sorry": 98, + "sort": 1129, + "sorting": 3617, + "sory": 2484, + "soryda": 2485, + "soul": 3618, + "sound": 1031, + "sounds": 825, + "soup": 2486, + "source": 3619, + "south": 1130, + "sp": 1131, + "space": 1442, + "spanish": 2487, + "speak": 462, + "speaking": 3620, + "special": 272, + "specially": 1443, + "specific": 3621, + "speechless": 3622, + "speed": 3623, + "speedchat": 3624, + "spell": 2488, + "spend": 1260, + "spending": 2489, + "spent": 1444, + "spider": 3625, + "spk": 2490, + "spl": 1943, + "splleing": 3626, + "spoiled": 3627, + "spoke": 1445, + "spoken": 2491, + "spook": 1628, + "sport": 1629, + "spree": 1261, + "sptv": 3628, + "sry": 2492, + "st": 949, + "stand": 1630, + "standard": 1446, + "standing": 3629, + "star": 3630, + "staring": 3631, + "stars": 2493, + "start": 347, + "started": 629, + "starting": 1032, + "starts": 1262, + "starwars3": 3632, + "statement": 826, + "station": 1631, + "stay": 630, + "stayed": 3633, + "staying": 1033, + "stays": 3634, + "std": 950, + "steam": 3635, + "step": 1944, + "steve": 3636, + "stick": 3637, + "sticky": 3638, + "still": 103, + "stock": 2494, + "stockport": 1447, + "stomach": 3639, + "stomps": 2495, + "stone": 3640, + "stones": 3641, + "stop": 100, + "stopped": 1945, + "stops": 3642, + "stopsms": 3643, + "store": 951, + "stores": 3644, + "storming": 2496, + "story": 631, + "str": 3645, + "straight": 1448, + "stranger": 2497, + "street": 1132, + "stretch": 3646, + "strong": 1632, + "stuck": 2498, + "student": 3647, + "students": 2499, + "study": 1133, + "studying": 1134, + "stuff": 376, + "stupid": 952, + "style": 1263, + "stylish": 1946, + "sub": 1264, + "subpoly": 3648, + "subs": 1947, + "subs16": 3649, + "subscribed": 3650, + "subscriber": 3651, + "subscription": 1948, + "successful": 2500, + "successfully": 3652, + "sucks": 1265, + "sue": 3653, + "sugar": 2501, + "suggest": 2502, + "suite": 1949, + "suite342": 874, + "sum": 3654, + "sum1": 2503, + "summer": 1034, + "sumthin": 3655, + "sun": 783, + "sunday": 875, + "sunny": 1633, + "sunshine": 1634, + "suntec": 2504, + "sup": 1635, + "super": 1950, + "superb": 3656, + "superior": 3657, + "supply": 2505, + "support": 1035, + "suppose": 3658, + "supposed": 1135, + "suprman": 3659, + "sura": 1951, + "sure": 206, + "surely": 1636, + "surfing": 1637, + "surprise": 827, + "surprised": 1952, + "survey": 3660, + "sux": 3661, + "suzy": 3662, + "sw7": 3663, + "swatch": 3664, + "sweet": 418, + "sweetest": 2506, + "sweetheart": 3665, + "sweets": 3666, + "swimming": 2507, + "swing": 1449, + "swiss": 3667, + "switch": 3668, + "swoop": 3669, + "swt": 1953, + "swtheart": 2508, + "symbol": 2509, + "system": 1638, + "t": 146, + "t's": 1136, + "ta": 1954, + "tablets": 2510, + "tacos": 3670, + "tahan": 3671, + "take": 121, + "taken": 2511, + "takes": 828, + "takin": 1955, + "taking": 736, + "talent": 3672, + "talk": 348, + "talking": 1036, + "tampa": 1137, + "tank": 3673, + "tariffs": 1639, + "tat": 1640, + "taunton": 1956, + "tayseer": 3674, + "tb": 1957, + "tc": 953, + "tcs": 1958, + "tea": 1266, + "teach": 1641, + "teacher": 3675, + "teaches": 1959, + "team": 1267, + "tear": 1450, + "tease": 2512, + "teasing": 1451, + "tech": 3676, + "technical": 2513, + "teeth": 1960, + "tel": 954, + "telephone": 3677, + "tell": 128, + "telling": 737, + "tells": 1961, + "telly": 3678, + "telphone": 2514, + "telugu": 3679, + "temple": 1962, + "ten": 876, + "tenants": 1963, + "tenerife": 1268, + "term": 2515, + "terms": 1037, + "terrible": 3680, + "terrorist": 3681, + "tessy": 3682, + "test": 571, + "testing": 3683, + "tests": 3684, + "text": 86, + "textbuddy": 3685, + "textcomp": 2516, + "texted": 3686, + "texting": 1964, + "textoperator": 2517, + "textpod": 1965, + "texts": 436, + "th": 955, + "than": 340, + "thangam": 2518, + "thank": 419, + "thanks": 195, + "thanksgiving": 2519, + "thanx": 393, + "that": 30, + "that'll": 3687, + "that's": 287, + "thats": 314, + "the": 12, + "theatre": 1642, + "their": 1138, + "them": 191, + "themob": 1966, + "then": 74, + "theory": 3688, + "there": 82, + "there're": 3689, + "there's": 543, + "theres": 3690, + "these": 592, + "thesis": 3691, + "they": 120, + "they're": 1452, + "thgt": 3692, + "thing": 241, + "things": 266, + "think": 133, + "thinkin": 1967, + "thinking": 593, + "thinks": 738, + "this": 56, + "thk": 267, + "thm": 3693, + "thnk": 1643, + "tho": 693, + "those": 486, + "thot": 3694, + "though": 505, + "thought": 349, + "thoughts": 3695, + "threats": 2520, + "three": 2521, + "throat": 1968, + "through": 956, + "throw": 3696, + "thru": 1269, + "ths": 2522, + "tht": 1453, + "thts": 1644, + "thurs": 1645, + "thursday": 1646, + "tick": 1454, + "ticket": 2523, + "tickets": 957, + "tihs": 1969, + "til": 632, + "till": 544, + "time": 78, + "times": 633, + "timing": 1647, + "tired": 1139, + "tirupur": 3697, + "tis": 1970, + "tissco": 3698, + "tiwary": 3699, + "tkts": 1971, + "tlp": 2524, + "tm": 1972, + "tmr": 463, + "tmrw": 3700, + "tncs": 1648, + "to": 6, + "toa": 2525, + "toclaim": 1455, + "today": 101, + "today's": 1456, + "todays": 666, + "tog": 2526, + "together": 694, + "tok": 3701, + "told": 288, + "toll": 2527, + "tom": 3702, + "tomarrow": 3703, + "tomo": 667, + "tomorro": 3704, + "tomorrow": 164, + "tone": 309, + "tones": 594, + "tones2you": 3705, + "tonight": 242, + "tonights": 3706, + "tonite": 1140, + "tons": 3707, + "too": 159, + "took": 695, + "tool": 3708, + "tooo": 3709, + "top": 739, + "topic": 3710, + "torch": 1457, + "tortilla": 3711, + "toshiba": 3712, + "tot": 521, + "total": 3713, + "totally": 1973, + "touch": 668, + "tough": 1270, + "toughest": 3714, + "tour": 1458, + "towards": 1459, + "town": 464, + "track": 1649, + "trade": 3715, + "train": 1141, + "training": 1271, + "transaction": 2528, + "transfer": 2529, + "transfered": 3716, + "transfr": 3717, + "transport": 3718, + "trav": 3719, + "travel": 1650, + "treat": 784, + "tree": 1974, + "tried": 522, + "trip": 740, + "trouble": 1272, + "true": 506, + "truffles": 3720, + "truly": 2530, + "trust": 1460, + "truth": 1038, + "try": 289, + "trying": 335, + "ts": 958, + "tsandcs": 3721, + "tscs": 2531, + "tscs087147403231winawk": 2532, + "tsunamis": 3722, + "tt": 2533, + "ttyl": 1975, + "tues": 3723, + "tuesday": 1461, + "tuition": 1651, + "tulip": 3724, + "turn": 1976, + "turning": 3725, + "turns": 2534, + "tv": 377, + "twelve": 2535, + "twenty": 3726, + "twice": 1977, + "two": 487, + "txt": 90, + "txt82228": 3727, + "txtauction": 1652, + "txtin": 3728, + "txting": 1273, + "txtno": 3729, + "txts": 877, + "tyler": 1978, + "type": 829, + "tyrone": 1979, + "u": 13, + "u'll": 1653, + "u're": 1654, + "u've": 2536, + "u4": 2537, + "ubi": 2538, + "ugh": 878, + "uh": 3730, + "uk": 243, + "uk's": 1462, + "uks": 2539, + "ultimatum": 3731, + "umma": 2540, + "ummmmmaah": 2541, + "un": 959, + "unable": 1655, + "unbreakable": 3732, + "uncle": 879, + "uncles": 2542, + "under": 2543, + "understand": 960, + "understanding": 2544, + "understood": 2545, + "unemployed": 3733, + "unfortunately": 3734, + "uni": 1980, + "unique": 3735, + "university": 2546, + "unkempt": 3736, + "unless": 1274, + "unlimited": 785, + "unnecessarily": 3737, + "unredeemed": 2547, + "unsold": 1981, + "unsub": 1656, + "unsubscribe": 669, + "untamed": 3738, + "until": 523, + "up": 58, + "update": 634, + "upgrade": 2548, + "upload": 2549, + "upset": 1982, + "upto": 2550, + "ur": 46, + "urawinner": 1463, + "ure": 1983, + "urgent": 207, + "urgently": 3739, + "urgnt": 1984, + "url": 1985, + "urn": 1986, + "urself": 1464, + "us": 226, + "usb": 3740, + "usc": 3741, + "use": 276, + "used": 880, + "useful": 3742, + "user": 1275, + "usf": 961, + "using": 1465, + "usual": 1142, + "usually": 1657, + "v": 290, + "vaazhthukkal": 3743, + "valentine": 1276, + "valentines": 1277, + "valid": 545, + "valid12hrs": 3744, + "valuable": 3745, + "value": 3746, + "valued": 881, + "various": 2551, + "vary": 1466, + "vday": 3747, + "ve": 1278, + "vegas": 3748, + "vegetables": 3749, + "verified": 3750, + "very": 192, + "vewy": 3751, + "via": 1143, + "vid": 3752, + "video": 378, + "videochat": 3753, + "videophones": 3754, + "vijay": 2552, + "vikky": 1658, + "village": 1987, + "violated": 3755, + "violence": 3756, + "violet": 3757, + "vip": 1988, + "virgin": 3758, + "visionsms": 3759, + "visit": 882, + "viva": 3760, + "vl": 1467, + "voda": 1659, + "vodafone": 1660, + "vodka": 3761, + "voice": 1144, + "voicemail": 2553, + "vomit": 3762, + "vomiting": 3763, + "vote": 2554, + "voucher": 670, + "vouchers": 635, + "vry": 1661, + "vth": 3764, + "vu": 3765, + "w": 671, + "w1": 3766, + "w111wx": 1989, + "w1j": 3767, + "w1j6hl": 1039, + "w1jhl": 3768, + "w45wq": 2555, + "wah": 2556, + "waht": 3769, + "wait": 231, + "waited": 3770, + "waitin": 1990, + "waiting": 277, + "wake": 524, + "waking": 1662, + "wales": 2557, + "walk": 741, + "walked": 3771, + "walking": 1663, + "wallpaper": 2558, + "walls": 3772, + "wan": 273, + "wan2": 3773, + "wana": 1145, + "wanna": 350, + "want": 88, + "wanted": 507, + "wanting": 3774, + "wants": 595, + "wap": 1279, + "warm": 1146, + "warner": 2559, + "warning": 3775, + "was": 79, + "wasn": 3776, + "wasn't": 1040, + "waste": 1664, + "wat": 168, + "wat's": 1280, + "watch": 420, + "watching": 379, + "water": 1041, + "wats": 2560, + "wave": 3777, + "waves": 3778, + "way": 162, + "wc1n3xx": 1991, + "we": 52, + "we'd": 1992, + "we'll": 672, + "we're": 696, + "we've": 1993, + "weak": 1994, + "wear": 2561, + "wearing": 2562, + "weather": 1468, + "website": 1995, + "wed": 1996, + "wedding": 2563, + "wednesday": 1469, + "wee": 3779, + "weed": 962, + "week": 150, + "week's": 2564, + "weekend": 508, + "weekends": 1281, + "weekly": 673, + "weeks": 883, + "weigh": 3780, + "weight": 1470, + "weird": 1997, + "welcome": 786, + "well": 139, + "welp": 3781, + "wen": 596, + "went": 244, + "wer": 2565, + "were": 232, + "west": 3782, + "westlife": 3783, + "wet": 1665, + "what": 68, + "what's": 572, + "whatever": 787, + "whats": 788, + "whatsup": 3784, + "when": 61, + "when's": 3785, + "whenever": 884, + "whenevr": 3786, + "where": 154, + "where's": 1666, + "wherever": 1998, + "whether": 3787, + "which": 260, + "while": 525, + "white": 1999, + "whn": 3788, + "who": 136, + "who's": 1471, + "whole": 789, + "whom": 2566, + "whos": 2567, + "whose": 3789, + "whr": 3790, + "why": 186, + "wid": 830, + "wif": 488, + "wife": 448, + "wifi": 3791, + "wihtuot": 3792, + "wil": 636, + "will": 49, + "willing": 2000, + "win": 196, + "wind": 3793, + "windows": 3794, + "wine": 1282, + "winner": 790, + "winning": 3795, + "wins": 2001, + "wipro": 3796, + "wisdom": 2568, + "wise": 2569, + "wish": 274, + "wishes": 1283, + "wishing": 1284, + "wiskey": 3797, + "wit": 963, + "with": 45, + "within": 885, + "without": 489, + "wiv": 1667, + "wk": 315, + "wkend": 2002, + "wkent": 3798, + "wkly": 742, + "wn": 3799, + "wnt": 2570, + "woke": 1147, + "woman": 2003, + "women": 2571, + "won": 216, + "won't": 597, + "wonder": 964, + "wonderful": 697, + "wondering": 1042, + "wonders": 2572, + "wont": 421, + "woot": 3800, + "word": 406, + "words": 598, + "work": 157, + "workin": 1285, + "working": 465, + "works": 1668, + "world": 407, + "worlds": 3801, + "worried": 1286, + "worries": 1287, + "worry": 546, + "worse": 2004, + "worth": 698, + "wot": 547, + "would": 201, + "wouldn't": 1669, + "wow": 1043, + "wrc": 3802, + "write": 1288, + "wrk": 2573, + "wrnog": 3803, + "wrong": 886, + "wt": 2574, + "wtf": 2005, + "wud": 2575, + "wun": 1670, + "www": 163, + "wylie": 2006, + "x": 284, + "x49": 3804, + "xam": 3805, + "xavier": 2576, + "xchat": 2007, + "xmas": 490, + "xuhui": 3806, + "xx": 637, + "xxx": 380, + "xxxx": 1671, + "xxxxx": 3807, + "xy": 1044, + "y": 295, + "ya": 259, + "yahoo": 1472, + "yan": 1672, + "yar": 791, + "yarasu": 3808, + "yay": 1673, + "yeah": 178, + "year": 301, + "years": 509, + "yeh": 3809, + "yep": 965, + "yer": 2008, + "yes": 169, + "yest": 1289, + "yesterday": 573, + "yet": 296, + "yetunde": 2009, + "yijue": 1290, + "ym": 2010, + "yo": 361, + "yoga": 1674, + "yogasana": 2011, + "yor": 2577, + "you": 7, + "you'd": 1675, + "you'll": 831, + "you're": 297, + "you've": 887, + "your": 23, + "youre": 3810, + "yours": 638, + "yourself": 699, + "yr": 966, + "yrs": 1148, + "yummy": 2578, + "yun": 1676, + "yunny": 3811, + "yuo": 2012, + "yup": 310, + "zed": 1677, + "|": 316, + "‘": 3812, + "’": 325, + "“": 3813, + "”": 3814, + "–": 2579, + "£": 55, + "é": 2013, + "ü": 104, + "ü'll": 3815, + "–": 1045, + "‘": 362, + "“": 3816 +} diff --git a/pytorch/pytorch_extended_tests/just_for_windows/run_pytorch_extended_tests.ps1 b/pytorch/pytorch_extended_tests/just_for_windows/run_pytorch_extended_tests.ps1 new file mode 100644 index 00000000..a1753f32 --- /dev/null +++ b/pytorch/pytorch_extended_tests/just_for_windows/run_pytorch_extended_tests.ps1 @@ -0,0 +1,50 @@ +$ErrorActionPreference = "Stop" + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$PythonBin = if ($env:PYTHON_BIN) { $env:PYTHON_BIN } else { "python" } +$ResultsDir = Join-Path ([System.IO.Path]::GetTempPath()) "ci_benchmarks\pytorch" + +Set-Location $RepoRoot + +# Start clean so the copied result folder only contains this run + +if (Test-Path $ResultsDir) { + Remove-Item -Recurse -Force $ResultsDir +} +New-Item -ItemType Directory -Force -Path $ResultsDir | Out-Null + +# Keep both the src package and root config package importable + +$PathSeparator = [System.IO.Path]::PathSeparator +$LocalPythonPath = "$RepoRoot\src$PathSeparator$RepoRoot" +if ($env:PYTHONPATH) { + $env:PYTHONPATH = "$LocalPythonPath$PathSeparator$env:PYTHONPATH" +} +else { + $env:PYTHONPATH = $LocalPythonPath +} + +Write-Host "Running pytorch_extended_tests" +Write-Host "Writing results to $ResultsDir\" + +# Windows PowerShell turns native stderr into ErrorRecord objects + +# Keep warnings in the log without treating them as terminating PowerShell errors + +$PreviousErrorActionPreference = $ErrorActionPreference +$ErrorActionPreference = "Continue" +try { + & $PythonBin -u -m pytorch_extended_tests.orchestrator.run_suite ` + --results-dir $ResultsDir ` + --keep-existing ` + @args 2>&1 | + ForEach-Object { $_.ToString() } | + Tee-Object -FilePath (Join-Path $ResultsDir "execution.log") + + $SuiteExitCode = $LASTEXITCODE +} +finally { + $ErrorActionPreference = $PreviousErrorActionPreference +} +Write-Host "Results are available in $ResultsDir\" +exit $SuiteExitCode \ No newline at end of file diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/README.md b/pytorch/pytorch_extended_tests/manual_comparison_stuff/README.md new file mode 100644 index 00000000..3d65fb09 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/README.md @@ -0,0 +1,117 @@ +# Manual comparison tools + +These scripts are for after CI has finished a few times and someone copied the raw result folders to their local. + +CI does not import or run these files + +The detailed comparisons work at output-leaf level (leaf = one final scalar, exact value or tensor reached after recursively flattening a stored output). This means a model state can be checked parameter by parameter rather than being reduced to one result. + +note - on training, it's steps rater than epochs, to make things not take so long. If everything looks good, we can increase it (maybe not run every night?). + +## Files + +- `level_0_first_look.py`: quick comparison of the small Level 0 CSV summaries + - Input: a `level_0_summaries/` folder containing `reference.csv` and one CSV per candidate environment + - Output: `level_0_first_look_collated.md` and `level_0_first_look_summary.md` in the input folder + - This is deliberately rough and does not replace the tensor comparison + +- `analyse_repeatability.py`: checks several runs from one environment against each other + - Input: a folder containing at least two `repeatability_*` subfolders, each an unmodified raw suite result bundle + - Output: `repeatability_analysis/repeatability_analysis.json`, a Markdown report and some PNG graphs + - Add `--write-populated-policy` for reference runs to also write `comparison_policy.json` + +- `comparison_policy_template.json`: central comparison-policy template + - Contains the exact-match rules, dtype-specific numerical floors and hard ceilings + - Reference repeatability fills the per-output and per-leaf calibration entries without replacing hardcoded values + +- `comparison_policy.py`: shared policy code + - Used by the repeatability analyser and both comparison scripts + - Handles policy loading, calibration, policy lookup and PASS/MAYBE/FAIL/NA judgements + +- `compare_repeatability_analyses.py`: compares already-created repeatability JSON files + - Input: `repeatability_outputs/reference.json` plus one or more candidate JSON files in the same folder + - Output: `repeatability_outputs/repeatability_comparison/` containing JSON, Markdown and an optional PNG summary + - It plots representative model losses and sampled final logits, plus Level 6 evaluation loss/accuracy + - A changed tensor hash is normally MAYBE here because the raw tensor values are not in the analysis JSON + +- `compare_environment_outputs.py`: full raw tensor-level comparison against the reference environment + - Input: one root folder containing `reference/` and one folder per candidate environment + - Each environment folder contains one or more raw suite result-bundle subfolders; their names do not matter + - Output: a populated `comparison_policy.json` plus `comparison_results/` containing JSON, Markdown and optional PNG graphs + - This is the main comparison when I need an actual numerical judgement + +## Repeatability input + +```text +collected_runs/ +├── repeatability_run_001/ +├── repeatability_run_002/ +└── repeatability_run_003/ +``` + +```bash +python manual_comparison_stuff/analyse_repeatability.py collected_runs --write-populated-policy +``` + +## Repeatability-JSON comparison input + +```text +repeatability_outputs/ +├── reference.json +├── gfx1201_scale_fp32.json +└── gfx_1100_scale_fp32.json +``` + +```bash +python manual_comparison_stuff/compare_repeatability_analyses.py repeatability_outputs \ + --policy manual_comparison_stuff/comparison_policy.json +``` + +## Raw environment comparison input + +```text +comparison_root/ +├── comparison_policy_template.json +├── reference/ +│ ├── run_001/ +│ └── run_002/ +├── gfx1201_scale_fp32/ +│ ├── run_001/ +│ └── run_002/ +└── gfx_1100_scale_fp32/ + ├── run_001/ + └── run_002/ +``` + +```bash +python manual_comparison_stuff/compare_environment_outputs.py comparison_root +``` + +The reference and candidate runs should use the same suite version, seed, prepared datasets, levels and profiles. A comparison across deliberately different profile sets will normally be NA or fail the compatibility checks + + + +## Training and inference graphs + +- Level 0 and Level 5 use two short optimisation steps +- The current Level 6 workloads are step-limited rather than epoch-limited +- The reports therefore show the first five configured evaluation checkpoints, with the real optimisation-step numbers on the x-axis +- They also show training loss over the first configured number of optimisation steps +- The detailed comparison reads the full final `checkpoint_logits` artefacts +- The summary comparison uses a deterministic compact preview stored in each repeatability-analysis JSON +- True five-epoch plots would require changing the Level 6 workload duration, especially for Fashion-MNIST and the Transformer + +Graph limits can be changed in the `reporting` section of `comparison_policy_template.json` without changing the tests + +## Reading the graphs + +- Every graph includes a title, axis labels and a short explanation on the image +- Reference training curves are labelled `reference baseline` +- Shaded bands on detailed training graphs are the minimum-to-maximum range across repeat runs +- Final-logit scatter plots use the reference on the x-axis and candidates on the y-axis +- The dashed `y = x` line in a logit scatter is exact agreement +- Final-logit error plots show candidate error from the reference and include a labelled zero-error reference baseline +- Prediction-disagreement plots include the reference self-comparison explicitly at `0%` +- Accuracy and disagreement axes are formatted as percentages +- A tolerance ratio of `1` is the pass boundary; values above `1` exceed at least one policy limit +- Repeatability error charts use logarithmic axes because the observed numerical differences can span many orders of magnitude \ No newline at end of file diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/analyse_repeatability.py b/pytorch/pytorch_extended_tests/manual_comparison_stuff/analyse_repeatability.py new file mode 100644 index 00000000..73d13d19 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/analyse_repeatability.py @@ -0,0 +1,1665 @@ +#!/usr/bin/env python3 +"""Analyse repeatability across raw suite result bundles. + +The input directory should contain two or more subdirectories whose names begin +with ``repeatability_``. Each of those subdirectories must be one unmodified +result bundle produced by this repository. + +The script measures observed variation first, then can optionally populate the +central comparison-policy template from the reference variability it observed. +The three top-level repeatability classifications mean: + +* ``exact``: every run produced the same output structure and every stored value + matched exactly, including tensor bytes and exceptional-value positions +* ``variable``: all outputs remained structurally comparable, but at least one + floating-point scalar or tensor changed numerically between runs. This is an + observation only and is not automatically a failure +* ``inconsistent``: at least one output was missing, failed to produce, changed + structure/dtype/shape, changed an exact value, or changed its NaN/Inf masks. + These differences cannot be treated as ordinary floating-point drift + +The JSON output is deliberately more detailed than the Markdown report. It keeps +stable output identities, per-run tensor artefact references, pairwise metrics, +variability summaries and representative-run choices so a later tool can compare +one GPU/compiler environment against another without rerunning this analysis. +""" + +from __future__ import annotations + +import argparse +import hashlib +import itertools +import json +import math +import statistics +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +import numpy as np + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPOSITORY_ROOT / "src" +for import_root in (REPOSITORY_ROOT, SRC_ROOT): + value = str(import_root) + if value not in sys.path: + sys.path.insert(0, value) + +from config.test_catalogue import get_test_spec # noqa: E402 +from comparison_policy import ( # noqa: E402 + BUILTIN_TEMPLATE_PATH, + load_policy_template, + populate_policy_from_repeatability, + write_json as write_policy_json, +) + + +ANALYSIS_FORMAT_VERSION = "repeatability_v1" +DEFAULT_JSON_NAME = "repeatability_analysis.json" +DEFAULT_MARKDOWN_NAME = "repeatability_analysis.md" +NUMERIC_RECORD_OUTPUT_IDS = {"summary", "final_metrics"} +PREVIEW_OUTPUT_IDS = {"loss_series", "training_loss", "checkpoint_metrics", "checkpoint_logits", "evaluation_outputs", "final_predictions"} +PREVIEW_TENSOR_VALUE_COUNT = 512 +CRITICAL_MANIFEST_FIELDS = ( + "suite_name", + "suite_version", + "result_format_version", + "test_catalogue_version", + "root_seed", + "dataset_manifest_sha256", + "device", + "profile_ids", +) + + +@dataclass(frozen=True, slots=True) +class RunBundle: + """One raw suite result bundle used as a repeat.""" + + run_id: str + path: Path + manifest: Mapping[str, Any] + tasks: tuple[Mapping[str, Any], ...] + observations: Mapping[str, Mapping[str, Any]] + + +@dataclass(frozen=True, slots=True) +class LeafValue: + """One recursively flattened value from an observation payload.""" + + path: str + value_type: str + value: Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input_directory", + nargs="?", + type=Path, + default=Path.cwd(), + help="Directory containing repeatability_* result-bundle subdirectories", + ) + parser.add_argument( + "--output-directory", + type=Path, + help="Output directory, defaulting to /repeatability_analysis", + ) + parser.add_argument( + "--pattern", + default="repeatability_*", + help="Glob used to locate result-bundle directories", + ) + parser.add_argument( + "--skip-artifact-hash-check", + action="store_true", + help="Trust tensor descriptor hashes instead of recalculating them", + ) + parser.add_argument( + "--no-plots", + action="store_true", + help="Do not produce the Matplotlib PNG summaries", + ) + parser.add_argument( + "--top-output-count", + type=int, + default=30, + help="Maximum number of variable outputs shown in the detailed Markdown table", + ) + parser.add_argument( + "--write-populated-policy", + nargs="?", + const="comparison_policy.json", + help=( + "Populate the comparison policy from this repeatability analysis. " + "With no path, writes comparison_policy.json in the analysis output directory" + ), + ) + parser.add_argument( + "--policy-template", + type=Path, + default=BUILTIN_TEMPLATE_PATH, + help="Comparison-policy template used with --write-populated-policy", + ) + return parser.parse_args() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise RuntimeError(f"Required file is missing: {path}") from exc + except json.JSONDecodeError as exc: + raise RuntimeError(f"File is not valid JSON: {path}") from exc + + +def read_json_lines(path: Path) -> list[dict[str, Any]]: + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError as exc: + raise RuntimeError(f"Required file is missing: {path}") from exc + + records: list[dict[str, Any]] = [] + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Invalid JSON on {path}:{line_number}") from exc + if not isinstance(value, dict): + raise RuntimeError(f"Observation on {path}:{line_number} is not an object") + records.append(value) + return records + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def observation_identity(record: Mapping[str, Any]) -> dict[str, Any]: + return { + "test_id": str(record.get("test_id")), + "case_id": str(record.get("case_id")), + "profile_id": str(record.get("profile_id")), + "output_id": str(record.get("output_id")), + "coordinates": record.get("coordinates"), + } + + +def observation_key(record: Mapping[str, Any]) -> str: + return canonical_json(observation_identity(record)) + + +def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def discover_run_directories(input_directory: Path, pattern: str) -> list[Path]: + candidates = [] + for path in sorted(input_directory.glob(pattern), key=lambda item: item.name): + if not path.is_dir(): + continue + if not (path / "run_manifest.json").is_file(): + continue + if not (path / "observations.jsonl").is_file(): + continue + if not (path / "test_status.json").is_file(): + continue + candidates.append(path) + if len(candidates) < 2: + raise RuntimeError( + f"Expected at least two valid {pattern!r} result directories beneath " + f"{input_directory}, found {len(candidates)}" + ) + return candidates + + +def load_run_bundle(path: Path) -> RunBundle: + manifest = read_json(path / "run_manifest.json") + status = read_json(path / "test_status.json") + tasks = status.get("tasks") if isinstance(status, Mapping) else None + if not isinstance(tasks, list): + raise RuntimeError(f"test_status.json has no tasks list: {path}") + + observations: dict[str, Mapping[str, Any]] = {} + for record in read_json_lines(path / "observations.jsonl"): + key = observation_key(record) + if key in observations: + raise RuntimeError(f"Duplicate observation identity in {path.name}: {key}") + observations[key] = record + + return RunBundle( + run_id=path.name, + path=path, + manifest=manifest, + tasks=tuple(tasks), + observations=observations, + ) + + +def compatibility_summary(runs: Sequence[RunBundle]) -> dict[str, Any]: + fields: dict[str, Any] = {} + mismatches: list[str] = [] + for field in CRITICAL_MANIFEST_FIELDS: + values = {run.run_id: run.manifest.get(field) for run in runs} + encoded = {canonical_json(value) for value in values.values()} + matches = len(encoded) == 1 + fields[field] = { + "matches": matches, + "common_value": next(iter(values.values())) if matches else None, + "values_by_run": values, + } + if not matches: + mismatches.append(field) + return { + "status": "compatible" if not mismatches else "incompatible", + "mismatched_fields": mismatches, + "fields": fields, + } + + +def validate_compatibility(summary: Mapping[str, Any]) -> None: + mismatches = summary.get("mismatched_fields") + if mismatches: + raise RuntimeError( + "The repeatability bundles are not directly comparable. " + "Mismatched manifest fields: " + ", ".join(str(item) for item in mismatches) + ) + + +def is_tensor_descriptor(value: Any) -> bool: + return isinstance(value, Mapping) and value.get("artifact_type") == "tensor" + + +def is_special_float(value: Any) -> bool: + return ( + isinstance(value, Mapping) + and value.get("value_type") == "special_float" + and isinstance(value.get("value"), str) + ) + + +def pointer_component(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def flatten_payload( + value: Any, + *, + path: str = "", + numeric_scalars: bool = True, +) -> list[LeafValue]: + if is_tensor_descriptor(value): + return [LeafValue(path or "/", "tensor", value)] + if is_special_float(value): + return [LeafValue(path or "/", "special_float", value)] + if isinstance(value, Mapping): + leaves: list[LeafValue] = [] + for key in sorted(value): + child_path = f"{path}/{pointer_component(str(key))}" + leaves.extend( + flatten_payload( + value[key], + path=child_path, + numeric_scalars=numeric_scalars, + ) + ) + if not leaves: + leaves.append(LeafValue(path or "/", "exact_value", {})) + return leaves + if isinstance(value, list): + leaves = [] + for index, item in enumerate(value): + leaves.extend( + flatten_payload( + item, + path=f"{path}/{index}", + numeric_scalars=numeric_scalars, + ) + ) + if not leaves: + leaves.append(LeafValue(path or "/", "exact_value", [])) + return leaves + if isinstance(value, bool) or value is None or isinstance(value, str): + return [LeafValue(path or "/", "exact_value", value)] + if isinstance(value, int): + return [LeafValue(path or "/", "exact_value", value)] + if isinstance(value, float): + value_type = "numeric_scalar" if numeric_scalars else "exact_value" + return [LeafValue(path or "/", value_type, value)] + return [LeafValue(path or "/", "exact_value", value)] + + +def descriptor_summary(descriptor: Mapping[str, Any]) -> dict[str, Any]: + fields = ( + "relative_path", + "sha256", + "byte_length", + "logical_dtype", + "storage_dtype", + "shape", + "numel", + "finite_count", + "nan_count", + "infinity_count", + "positive_infinity_count", + "negative_infinity_count", + ) + return {field: descriptor.get(field) for field in fields} + + +def _bfloat16_to_float32(values: np.ndarray) -> np.ndarray: + words = values.astype(np.uint32, copy=False) << np.uint32(16) + return words.view(np.float32) + + +def validate_tensor_artifact_file( + run: RunBundle, + descriptor: Mapping[str, Any], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> Path: + relative_path = descriptor.get("relative_path") + if not isinstance(relative_path, str): + raise RuntimeError(f"Tensor descriptor in {run.run_id} has no relative_path") + path = run.path / relative_path + if not path.is_file(): + raise RuntimeError(f"Tensor artefact is missing: {path}") + + expected_length = descriptor.get("byte_length") + if not isinstance(expected_length, int) or path.stat().st_size != expected_length: + raise RuntimeError(f"Tensor artefact size does not match its descriptor: {path}") + + cache_key = (run.path.as_posix(), relative_path) + if verify_hash and cache_key not in verified_paths: + expected_hash = descriptor.get("sha256") + if not isinstance(expected_hash, str) or hash_file(path) != expected_hash: + raise RuntimeError(f"Tensor artefact hash does not match: {path}") + verified_paths.add(cache_key) + return path + + +def load_tensor( + run: RunBundle, + descriptor: Mapping[str, Any], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> np.ndarray: + path = validate_tensor_artifact_file( + run, + descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + + storage_dtype = descriptor.get("storage_dtype") + logical_dtype = descriptor.get("logical_dtype") + shape = descriptor.get("shape") + numel = descriptor.get("numel") + if not isinstance(storage_dtype, str) or not isinstance(shape, list) or not isinstance(numel, int): + raise RuntimeError(f"Tensor descriptor is incomplete: {path}") + + stored = np.fromfile(path, dtype=np.dtype(storage_dtype), count=numel) + if stored.size != numel: + raise RuntimeError(f"Tensor artefact has the wrong element count: {path}") + if logical_dtype == "bfloat16": + values = _bfloat16_to_float32(stored.astype(np.uint16, copy=False)) + else: + values = stored + return values.reshape(tuple(int(item) for item in shape)) + + +def _safe_float(value: float | np.floating[Any]) -> float | None: + converted = float(value) + return converted if math.isfinite(converted) else None + + +def _percentile(values: Sequence[float], percentile: float) -> float | None: + if not values: + return None + return _safe_float(np.percentile(np.asarray(values, dtype=np.float64), percentile)) + + +def numeric_pair_metrics(left: np.ndarray, right: np.ndarray) -> dict[str, Any]: + if left.shape != right.shape: + return { + "comparable": False, + "reason": "shape_mismatch", + "left_shape": list(left.shape), + "right_shape": list(right.shape), + } + + left_values = np.asarray(left) + right_values = np.asarray(right) + left_nan = np.isnan(left_values) if np.issubdtype(left_values.dtype, np.inexact) else np.zeros(left.shape, dtype=bool) + right_nan = np.isnan(right_values) if np.issubdtype(right_values.dtype, np.inexact) else np.zeros(right.shape, dtype=bool) + left_inf = np.isinf(left_values) if np.issubdtype(left_values.dtype, np.inexact) else np.zeros(left.shape, dtype=bool) + right_inf = np.isinf(right_values) if np.issubdtype(right_values.dtype, np.inexact) else np.zeros(right.shape, dtype=bool) + left_finite = ~(left_nan | left_inf) + right_finite = ~(right_nan | right_inf) + jointly_finite = left_finite & right_finite + + nan_mask_mismatch = int(np.count_nonzero(left_nan != right_nan)) + infinity_mask_mismatch = int(np.count_nonzero(left_inf != right_inf)) + finite_mask_mismatch = int(np.count_nonzero(left_finite != right_finite)) + + exact_mask = left_values == right_values + exact_mask = exact_mask | (left_nan & right_nan) + mismatch_count = int(left_values.size - np.count_nonzero(exact_mask)) + + metrics: dict[str, Any] = { + "comparable": True, + "element_count": int(left_values.size), + "exact_equal": mismatch_count == 0, + "mismatch_count": mismatch_count, + "mismatch_fraction": mismatch_count / left_values.size if left_values.size else 0.0, + "nan_mask_mismatch_count": nan_mask_mismatch, + "infinity_mask_mismatch_count": infinity_mask_mismatch, + "finite_mask_mismatch_count": finite_mask_mismatch, + "jointly_finite_count": int(np.count_nonzero(jointly_finite)), + } + + if not np.any(jointly_finite): + metrics.update( + { + "maximum_absolute_error": None, + "mean_absolute_error": None, + "root_mean_square_error": None, + "normalised_root_mean_square_error": None, + "relative_l2_error": None, + "maximum_symmetric_relative_error": None, + } + ) + return metrics + + left_f = left_values[jointly_finite].astype(np.complex128 if np.iscomplexobj(left_values) else np.float64) + right_f = right_values[jointly_finite].astype(np.complex128 if np.iscomplexobj(right_values) else np.float64) + absolute_error = np.abs(left_f - right_f).astype(np.float64) + left_abs = np.abs(left_f).astype(np.float64) + right_abs = np.abs(right_f).astype(np.float64) + + difference_l2 = float(np.linalg.norm(absolute_error.ravel(), ord=2)) + left_l2 = float(np.linalg.norm(left_abs.ravel(), ord=2)) + right_l2 = float(np.linalg.norm(right_abs.ravel(), ord=2)) + relative_l2 = difference_l2 / max(left_l2, right_l2, np.finfo(np.float64).tiny) + + rmse = float(np.sqrt(np.mean(np.square(absolute_error, dtype=np.float64)))) + left_rms = float(np.sqrt(np.mean(np.square(left_abs, dtype=np.float64)))) + right_rms = float(np.sqrt(np.mean(np.square(right_abs, dtype=np.float64)))) + normalised_rmse = rmse / max(left_rms, right_rms, np.finfo(np.float64).tiny) + + symmetric_denominator = np.maximum( + np.maximum(left_abs, right_abs), + np.finfo(np.float64).tiny, + ) + symmetric_relative = absolute_error / symmetric_denominator + metrics.update( + { + "maximum_absolute_error": _safe_float(np.max(absolute_error)), + "mean_absolute_error": _safe_float(np.mean(absolute_error)), + "root_mean_square_error": _safe_float(rmse), + "normalised_root_mean_square_error": _safe_float(normalised_rmse), + "relative_l2_error": _safe_float(relative_l2), + "maximum_symmetric_relative_error": _safe_float(np.max(symmetric_relative)), + } + ) + return metrics + + +def tensor_pair_metrics( + left_run: RunBundle, + left_descriptor: Mapping[str, Any], + right_run: RunBundle, + right_descriptor: Mapping[str, Any], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> dict[str, Any]: + metadata_fields = ("logical_dtype", "shape", "numel") + metadata_matches = all(left_descriptor.get(field) == right_descriptor.get(field) for field in metadata_fields) + result: dict[str, Any] = { + "run_a": left_run.run_id, + "run_b": right_run.run_id, + "metadata_matches": metadata_matches, + "sha256_matches": left_descriptor.get("sha256") == right_descriptor.get("sha256"), + } + if not metadata_matches: + result.update( + { + "comparable": False, + "reason": "tensor_metadata_mismatch", + "left": descriptor_summary(left_descriptor), + "right": descriptor_summary(right_descriptor), + } + ) + return result + + validate_tensor_artifact_file( + left_run, + left_descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + validate_tensor_artifact_file( + right_run, + right_descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + + if result["sha256_matches"]: + result.update( + { + "comparable": True, + "element_count": int(left_descriptor.get("numel", 0)), + "exact_equal": True, + "mismatch_count": 0, + "mismatch_fraction": 0.0, + "nan_mask_mismatch_count": 0, + "infinity_mask_mismatch_count": 0, + "finite_mask_mismatch_count": 0, + "jointly_finite_count": int(left_descriptor.get("finite_count", 0)), + "maximum_absolute_error": 0.0, + "mean_absolute_error": 0.0, + "root_mean_square_error": 0.0, + "normalised_root_mean_square_error": 0.0, + "relative_l2_error": 0.0, + "maximum_symmetric_relative_error": 0.0, + } + ) + return result + + left = load_tensor( + left_run, + left_descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + right = load_tensor( + right_run, + right_descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + result.update(numeric_pair_metrics(left, right)) + return result + + +def scalar_pair_metrics(left: Any, right: Any, *, run_a: str, run_b: str) -> dict[str, Any]: + left_array = np.asarray([left]) + right_array = np.asarray([right]) + result = {"run_a": run_a, "run_b": run_b} + result.update(numeric_pair_metrics(left_array, right_array)) + return result + + +def exact_pair_result(left: Any, right: Any, *, run_a: str, run_b: str) -> dict[str, Any]: + return { + "run_a": run_a, + "run_b": run_b, + "comparable": True, + "exact_equal": canonical_json(left) == canonical_json(right), + } + + +def pair_numeric_distance(pair: Mapping[str, Any]) -> float | None: + value = pair.get("relative_l2_error") + if isinstance(value, (int, float)) and math.isfinite(float(value)): + return float(value) + value = pair.get("normalised_root_mean_square_error") + if isinstance(value, (int, float)) and math.isfinite(float(value)): + return float(value) + return None + + +def summarise_pairwise(pairwise: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + exact_pair_count = sum(bool(pair.get("exact_equal")) for pair in pairwise) + comparable_pair_count = sum(bool(pair.get("comparable")) for pair in pairwise) + numeric_distances = [ + value + for pair in pairwise + if (value := pair_numeric_distance(pair)) is not None + ] + absolute_errors = [ + float(value) + for pair in pairwise + if isinstance((value := pair.get("maximum_absolute_error")), (int, float)) + and math.isfinite(float(value)) + ] + mismatch_fractions = [ + float(value) + for pair in pairwise + if isinstance((value := pair.get("mismatch_fraction")), (int, float)) + and math.isfinite(float(value)) + ] + + worst_pair = None + if pairwise: + def ranking(pair: Mapping[str, Any]) -> tuple[int, int, float, float]: + structural = 0 if pair.get("comparable") else 1 + exact_mismatch = 0 if pair.get("exact_equal") else 1 + numeric = pair_numeric_distance(pair) or 0.0 + absolute = float(pair.get("maximum_absolute_error") or 0.0) + return structural, exact_mismatch, numeric, absolute + + worst_pair = dict(max(pairwise, key=ranking)) + + return { + "pair_count": len(pairwise), + "comparable_pair_count": comparable_pair_count, + "exact_pair_count": exact_pair_count, + "exact_pair_fraction": exact_pair_count / len(pairwise) if pairwise else None, + "maximum_relative_l2_error": max(numeric_distances, default=None), + "median_relative_l2_error": statistics.median(numeric_distances) if numeric_distances else None, + "p95_relative_l2_error": _percentile(numeric_distances, 95.0), + "maximum_absolute_error": max(absolute_errors, default=None), + "maximum_mismatch_fraction": max(mismatch_fractions, default=None), + "worst_pair": worst_pair, + } + + +def classify_leaf( + *, + run_count: int, + available_count: int, + value_type: str, + pairwise: Sequence[Mapping[str, Any]], +) -> str: + if available_count != run_count: + return "missing_runs" + if any(not pair.get("comparable", False) for pair in pairwise): + return "structural_mismatch" + if all(pair.get("exact_equal", False) for pair in pairwise): + return "exact" + if value_type in {"tensor", "numeric_scalar"}: + if any( + int(pair.get("finite_mask_mismatch_count", 0)) > 0 + or int(pair.get("nan_mask_mismatch_count", 0)) > 0 + or int(pair.get("infinity_mask_mismatch_count", 0)) > 0 + for pair in pairwise + ): + return "exceptional_value_mismatch" + return "numeric_variation" + return "exact_value_mismatch" + + +def leaf_distance(pair: Mapping[str, Any], value_type: str) -> float: + if not pair.get("comparable", False): + return 1_000_000.0 + if pair.get("exact_equal", False): + return 0.0 + if value_type in {"tensor", "numeric_scalar"}: + numeric = pair_numeric_distance(pair) + if numeric is not None: + return math.log1p(max(numeric, 0.0)) + return 10.0 + return 1.0 + + +def medoid_run(run_ids: Sequence[str], pair_distances: Mapping[tuple[str, str], float]) -> str | None: + if not run_ids: + return None + if len(run_ids) == 1: + return run_ids[0] + scores: dict[str, float] = {} + for run_id in run_ids: + score = 0.0 + for other in run_ids: + if other == run_id: + continue + key = tuple(sorted((run_id, other))) + score += float(pair_distances.get(key, 1_000_000.0)) + scores[run_id] = score + return min(run_ids, key=lambda item: (scores[item], item)) + + +def analyse_leaf( + *, + path: str, + values_by_run: Mapping[str, LeafValue], + runs_by_id: Mapping[str, RunBundle], + all_run_ids: Sequence[str], + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> dict[str, Any]: + value_types = sorted({leaf.value_type for leaf in values_by_run.values()}) + value_type = value_types[0] if len(value_types) == 1 else "mixed" + pairwise: list[dict[str, Any]] = [] + pair_distances: dict[tuple[str, str], float] = {} + + for run_a, run_b in itertools.combinations(sorted(values_by_run), 2): + left = values_by_run[run_a] + right = values_by_run[run_b] + if left.value_type != right.value_type: + pair = { + "run_a": run_a, + "run_b": run_b, + "comparable": False, + "exact_equal": False, + "reason": "value_type_mismatch", + "left_value_type": left.value_type, + "right_value_type": right.value_type, + } + elif left.value_type == "tensor": + pair = tensor_pair_metrics( + runs_by_id[run_a], + left.value, + runs_by_id[run_b], + right.value, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + elif left.value_type == "numeric_scalar": + pair = scalar_pair_metrics(left.value, right.value, run_a=run_a, run_b=run_b) + else: + pair = exact_pair_result(left.value, right.value, run_a=run_a, run_b=run_b) + pairwise.append(pair) + pair_distances[tuple(sorted((run_a, run_b)))] = leaf_distance(pair, left.value_type) + + run_values: dict[str, Any] = {} + for run_id, leaf in sorted(values_by_run.items()): + if leaf.value_type == "tensor": + run_values[run_id] = descriptor_summary(leaf.value) + else: + run_values[run_id] = leaf.value + + status = classify_leaf( + run_count=len(all_run_ids), + available_count=len(values_by_run), + value_type=value_type, + pairwise=pairwise, + ) + return { + "path": path, + "value_type": value_type, + "status": status, + "available_runs": sorted(values_by_run), + "missing_runs": sorted(set(all_run_ids) - set(values_by_run)), + "representative_run": medoid_run(sorted(values_by_run), pair_distances), + "runs": run_values, + "pairwise": pairwise, + "summary": summarise_pairwise(pairwise), + } + + + + +def _json_numeric(value: Any) -> Any: + """Convert one NumPy scalar into ordinary JSON data.""" + + converted = np.asarray(value).item() + if isinstance(converted, complex): + return {"real": float(converted.real), "imag": float(converted.imag)} + if isinstance(converted, (np.bool_, bool)): + return bool(converted) + if isinstance(converted, (np.integer, int)): + return int(converted) + if isinstance(converted, (np.floating, float)): + numeric = float(converted) + return numeric if math.isfinite(numeric) else None + return converted + + +def _tensor_preview(array: np.ndarray, descriptor: Mapping[str, Any]) -> dict[str, Any]: + """Keep a compact numeric preview without turning analysis JSON into another artefact bundle.""" + + values = np.asarray(array) + flattened = values.reshape(-1) + sample_count = min(flattened.size, PREVIEW_TENSOR_VALUE_COUNT) + if sample_count: + sample_indices = np.linspace(0, flattened.size - 1, sample_count, dtype=np.int64) + sample_values = [_json_numeric(flattened[index]) for index in sample_indices] + else: + sample_indices = np.asarray([], dtype=np.int64) + sample_values = [] + finite = np.isfinite(values) if np.issubdtype(values.dtype, np.inexact) else np.ones(values.shape, dtype=bool) + finite_values = values[finite] + if np.iscomplexobj(finite_values): + summary_values = np.abs(finite_values).astype(np.float64) + else: + summary_values = finite_values.astype(np.float64, copy=False) + return { + "logical_dtype": descriptor.get("logical_dtype"), + "shape": list(values.shape), + "numel": int(values.size), + "sample_indices": sample_indices.tolist(), + "sample_values": sample_values, + "mean": float(np.mean(summary_values)) if summary_values.size else None, + "standard_deviation": float(np.std(summary_values)) if summary_values.size else None, + "minimum": float(np.min(summary_values)) if summary_values.size else None, + "maximum": float(np.max(summary_values)) if summary_values.size else None, + "l2_norm": float(np.linalg.norm(summary_values.reshape(-1), 2)) if summary_values.size else 0.0, + } + + +def build_representative_preview( + value: Any, + *, + output_id: str, + run: RunBundle, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> Any: + """Load just enough representative data for the lightweight comparison plots.""" + + if is_tensor_descriptor(value): + array = load_tensor( + run, + value, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if array.size == 1: + return _json_numeric(array.reshape(-1)[0]) + if output_id in {"checkpoint_logits", "evaluation_outputs", "final_predictions"}: + return _tensor_preview(array, value) + return descriptor_summary(value) + if isinstance(value, Mapping): + return { + str(key): build_representative_preview( + item, + output_id=output_id, + run=run, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + for key, item in value.items() + } + if isinstance(value, list): + return [ + build_representative_preview( + item, + output_id=output_id, + run=run, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + for item in value + ] + return value + +def output_pair_distances( + leaves: Sequence[Mapping[str, Any]], + run_ids: Sequence[str], +) -> tuple[list[dict[str, Any]], dict[tuple[str, str], float]]: + output_pairs: list[dict[str, Any]] = [] + distances: dict[tuple[str, str], float] = {} + for run_a, run_b in itertools.combinations(sorted(run_ids), 2): + leaf_distances: list[float] = [] + structural_mismatch_count = 0 + exact_value_mismatch_count = 0 + numeric_values: list[float] = [] + compared_leaf_count = 0 + for leaf in leaves: + pair = next( + ( + item + for item in leaf.get("pairwise", []) + if {item.get("run_a"), item.get("run_b")} == {run_a, run_b} + ), + None, + ) + if pair is None: + structural_mismatch_count += 1 + leaf_distances.append(1_000_000.0) + continue + compared_leaf_count += 1 + if not pair.get("comparable", False): + structural_mismatch_count += 1 + elif not pair.get("exact_equal", False) and leaf.get("value_type") not in { + "tensor", + "numeric_scalar", + }: + exact_value_mismatch_count += 1 + value = pair_numeric_distance(pair) + if value is not None: + numeric_values.append(value) + leaf_distances.append(leaf_distance(pair, str(leaf.get("value_type")))) + + distance = max(leaf_distances, default=0.0) + distances[(run_a, run_b)] = distance + output_pairs.append( + { + "run_a": run_a, + "run_b": run_b, + "compared_leaf_count": compared_leaf_count, + "structural_mismatch_count": structural_mismatch_count, + "exact_value_mismatch_count": exact_value_mismatch_count, + "maximum_relative_l2_error": max(numeric_values, default=None), + "median_relative_l2_error": statistics.median(numeric_values) if numeric_values else None, + "distance_score": distance, + } + ) + return output_pairs, distances + + +def classify_output(run_count: int, records_by_run: Mapping[str, Mapping[str, Any]], leaves: Sequence[Mapping[str, Any]]) -> str: + # exact means the structure and all stored values match exactly across every run + # numeric_variation means only ordinary numeric values changed and remain comparable + # every other result is inconsistent because it involves missing data, structure or exact values + if len(records_by_run) != run_count: + return "missing_runs" + statuses = {str(record.get("status")) for record in records_by_run.values()} + if statuses == {"skipped_unsupported"}: + return "skipped_unsupported" + if statuses == {"failed_to_produce"}: + return "failed_to_produce" + if statuses != {"produced"}: + return "not_produced_consistently" + leaf_statuses = {str(leaf.get("status")) for leaf in leaves} + if leaf_statuses <= {"exact"}: + return "exact" + if leaf_statuses <= {"exact", "numeric_variation"}: + return "numeric_variation" + if "structural_mismatch" in leaf_statuses: + return "structural_mismatch" + if "missing_runs" in leaf_statuses: + return "missing_runs" + if "exceptional_value_mismatch" in leaf_statuses: + return "exceptional_value_mismatch" + return "value_mismatch" + + +def analyse_output( + key: str, + *, + records_by_run: Mapping[str, Mapping[str, Any]], + runs_by_id: Mapping[str, RunBundle], + all_run_ids: Sequence[str], + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> dict[str, Any]: + first_record = next(iter(records_by_run.values())) + identity = observation_identity(first_record) + try: + test_spec = get_test_spec(identity["test_id"]) + level = test_spec.level + category = test_spec.category + except KeyError: + level = "unknown" + category = "Unknown" + + flattened_by_run: dict[str, dict[str, LeafValue]] = {} + for run_id, record in records_by_run.items(): + if record.get("status") != "produced": + continue + flattened = flatten_payload( + record.get("payload"), + numeric_scalars=( + record.get("kind") in {"scalar", "series"} + or record.get("output_id") in NUMERIC_RECORD_OUTPUT_IDS + ), + ) + flattened_by_run[run_id] = {leaf.path: leaf for leaf in flattened} + + all_paths = sorted({path for leaves in flattened_by_run.values() for path in leaves}) + leaves: list[dict[str, Any]] = [] + for path in all_paths: + values_by_run = { + run_id: flattened[path] + for run_id, flattened in flattened_by_run.items() + if path in flattened + } + leaves.append( + analyse_leaf( + path=path, + values_by_run=values_by_run, + runs_by_id=runs_by_id, + all_run_ids=all_run_ids, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + ) + + output_pairs, distances = output_pair_distances(leaves, sorted(records_by_run)) + representative_run = medoid_run(sorted(records_by_run), distances) + representative_preview = None + if ( + representative_run is not None + and identity["output_id"] in PREVIEW_OUTPUT_IDS + and records_by_run[representative_run].get("status") == "produced" + ): + representative_preview = build_representative_preview( + records_by_run[representative_run].get("payload"), + output_id=identity["output_id"], + run=runs_by_id[representative_run], + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + status = classify_output(len(all_run_ids), records_by_run, leaves) + leaf_counts = Counter(str(leaf.get("status")) for leaf in leaves) + relative_l2_values = [ + float(value) + for leaf in leaves + if isinstance( + (value := leaf.get("summary", {}).get("maximum_relative_l2_error")), + (int, float), + ) + and math.isfinite(float(value)) + ] + absolute_values = [ + float(value) + for leaf in leaves + if isinstance( + (value := leaf.get("summary", {}).get("maximum_absolute_error")), + (int, float), + ) + and math.isfinite(float(value)) + ] + + return { + "identity_key": key, + "identity": identity, + "level": level, + "category": category, + "kind": first_record.get("kind"), + "importance": first_record.get("importance"), + "status": status, + "available_runs": sorted(records_by_run), + "missing_runs": sorted(set(all_run_ids) - set(records_by_run)), + "run_observation_statuses": { + run_id: record.get("status") for run_id, record in sorted(records_by_run.items()) + }, + "run_observations": { + run_id: { + "status": record.get("status"), + "reason": record.get("reason"), + "seed": record.get("seed"), + "kind": record.get("kind"), + "importance": record.get("importance"), + } + for run_id, record in sorted(records_by_run.items()) + }, + "representative_run": representative_run, + "representative_preview": representative_preview, + "leaf_status_counts": dict(sorted(leaf_counts.items())), + "maximum_relative_l2_error": max(relative_l2_values, default=None), + "maximum_absolute_error": max(absolute_values, default=None), + "pairwise_output_distances": output_pairs, + "leaves": leaves, + } + + +def run_summary(run: RunBundle, input_root: Path) -> dict[str, Any]: + task_statuses = Counter(str(task.get("status", "unknown")) for task in run.tasks) + observation_statuses = Counter( + str(record.get("status", "unknown")) for record in run.observations.values() + ) + return { + "run_id": run.run_id, + "bundle_relative_path": run.path.relative_to(input_root).as_posix(), + "overall_execution_status": run.manifest.get("overall_execution_status"), + "planned_task_count": run.manifest.get("planned_task_count"), + "task_count": len(run.tasks), + "task_status_counts": dict(sorted(task_statuses.items())), + "observation_count": len(run.observations), + "observation_status_counts": dict(sorted(observation_statuses.items())), + "started_at_utc": run.manifest.get("started_at_utc"), + "ended_at_utc": run.manifest.get("ended_at_utc"), + } + + +def aggregate_environment_pairs(outputs: Sequence[Mapping[str, Any]], run_ids: Sequence[str]) -> tuple[list[dict[str, Any]], str | None]: + pairs: list[dict[str, Any]] = [] + medoid_distances: dict[tuple[str, str], float] = {} + for run_a, run_b in itertools.combinations(sorted(run_ids), 2): + structural = 0 + exact_mismatches = 0 + numeric_values: list[float] = [] + compared_outputs = 0 + distance_values: list[float] = [] + for output in outputs: + pair = next( + ( + item + for item in output.get("pairwise_output_distances", []) + if {item.get("run_a"), item.get("run_b")} == {run_a, run_b} + ), + None, + ) + if pair is None: + structural += 1 + distance_values.append(1_000_000.0) + continue + compared_outputs += 1 + structural += int(pair.get("structural_mismatch_count", 0)) + exact_mismatches += int(pair.get("exact_value_mismatch_count", 0)) + numeric = pair.get("maximum_relative_l2_error") + if isinstance(numeric, (int, float)) and math.isfinite(float(numeric)): + numeric_values.append(float(numeric)) + distance_values.append(float(pair.get("distance_score", 0.0))) + + aggregate_distance = max(distance_values, default=0.0) + medoid_distances[(run_a, run_b)] = aggregate_distance + pairs.append( + { + "run_a": run_a, + "run_b": run_b, + "compared_output_count": compared_outputs, + "structural_mismatch_count": structural, + "exact_value_mismatch_count": exact_mismatches, + "maximum_relative_l2_error": max(numeric_values, default=None), + "median_relative_l2_error": statistics.median(numeric_values) if numeric_values else None, + "p95_relative_l2_error": _percentile(numeric_values, 95.0), + "distance_score": aggregate_distance, + } + ) + return pairs, medoid_run(sorted(run_ids), medoid_distances) + + +def build_analysis( + runs: Sequence[RunBundle], + *, + input_root: Path, + verify_hash: bool, +) -> dict[str, Any]: + compatibility = compatibility_summary(runs) + validate_compatibility(compatibility) + + runs_by_id = {run.run_id: run for run in runs} + run_ids = sorted(runs_by_id) + all_keys = sorted({key for run in runs for key in run.observations}) + verified_paths: set[tuple[str, str]] = set() + outputs = [] + for key in all_keys: + records_by_run = { + run.run_id: run.observations[key] + for run in runs + if key in run.observations + } + outputs.append( + analyse_output( + key, + records_by_run=records_by_run, + runs_by_id=runs_by_id, + all_run_ids=run_ids, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + ) + + output_status_counts = Counter(str(output.get("status")) for output in outputs) + leaf_status_counts = Counter( + str(leaf.get("status")) + for output in outputs + for leaf in output.get("leaves", []) + ) + environment_pairs, representative_run = aggregate_environment_pairs(outputs, run_ids) + + category_counts: dict[str, Counter[str]] = defaultdict(Counter) + profile_counts: dict[str, Counter[str]] = defaultdict(Counter) + for output in outputs: + category_counts[str(output.get("category"))][str(output.get("status"))] += 1 + identity = output.get("identity", {}) + profile_counts[str(identity.get("profile_id"))][str(output.get("status"))] += 1 + + inconsistent_statuses = { + "missing_runs", + "not_produced_consistently", + "failed_to_produce", + "structural_mismatch", + "exceptional_value_mismatch", + "value_mismatch", + } + overall_classification = ( + "inconsistent" + if any(status in inconsistent_statuses for status in output_status_counts) + else "variable" + if output_status_counts.get("numeric_variation", 0) + else "exact" + ) + + return { + "analysis_format_version": ANALYSIS_FORMAT_VERSION, + "analysis_kind": "intra_environment_repeatability", + "generated_at_utc": utc_now(), + "input_root": input_root.as_posix(), + "run_count": len(runs), + "run_order": run_ids, + "overall_classification": overall_classification, + "classification_note": ( + "This is an observed-variation classification, not a policy-based PASS or FAIL" + ), + "compatibility": compatibility, + "runs": [run_summary(runs_by_id[run_id], input_root) for run_id in run_ids], + "environment_representative_run": representative_run, + "environment_pairwise_distances": environment_pairs, + "summary": { + "output_count": len(outputs), + "output_status_counts": dict(sorted(output_status_counts.items())), + "leaf_status_counts": dict(sorted(leaf_status_counts.items())), + "category_status_counts": { + category: dict(sorted(counts.items())) + for category, counts in sorted(category_counts.items()) + }, + "profile_status_counts": { + profile: dict(sorted(counts.items())) + for profile, counts in sorted(profile_counts.items()) + }, + "verified_tensor_artifact_count": len(verified_paths) if verify_hash else None, + "artifact_hashes_verified": verify_hash, + }, + "outputs": outputs, + "plots": [], + } + + +def format_number(value: Any) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "yes" if value else "no" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value == 0.0: + return "0" + if abs(value) >= 1000 or abs(value) < 0.001: + return f"{value:.3e}" + return f"{value:.6g}" + return str(value) + + +def markdown_escape(value: Any) -> str: + return str(value).replace("|", "\\|").replace("\n", " ") + + +def write_markdown( + analysis: Mapping[str, Any], + path: Path, + *, + plot_paths: Sequence[Path], + top_output_count: int, +) -> None: + lines = [ + "# Repeatability analysis", + "", + ( + "This report measures variation between repeated runs of one environment. " + "It does not apply the later central numerical acceptance policy, so " + "`numeric_variation` is descriptive rather than a failure" + ), + "", + "## Overview", + "", + f"- Runs analysed: **{analysis['run_count']}**", + f"- Classification: **{analysis['overall_classification']}**", + f"- Representative run: **{analysis.get('environment_representative_run') or 'none'}**", + f"- Output observations: **{analysis['summary']['output_count']}**", + "", + "## Input runs", + "", + "| Run | Execution | Tasks | Observations | Started |", + "|---|---|---:|---:|---|", + ] + for run in analysis.get("runs", []): + lines.append( + "| " + + " | ".join( + [ + markdown_escape(run.get("run_id")), + markdown_escape(run.get("overall_execution_status")), + format_number(run.get("task_count")), + format_number(run.get("observation_count")), + markdown_escape(run.get("started_at_utc") or "—"), + ] + ) + + " |" + ) + + lines.extend( + [ + "", + "## Output classifications", + "", + "| Classification | Count | Meaning |", + "|---|---:|---|", + ] + ) + meanings = { + "exact": "Every repeat stored the same values or tensor bytes", + "numeric_variation": "Structures match, but at least one numeric value differs", + "exceptional_value_mismatch": "NaN, infinity or finite-value positions differ", + "value_mismatch": "A non-numeric value differs", + "structural_mismatch": "Payload structure, tensor shape or dtype differs", + "missing_runs": "The output or one of its leaves is absent from one or more runs", + "not_produced_consistently": "Runs disagree about whether the output was produced", + "failed_to_produce": "Every run failed before producing this output", + "skipped_unsupported": "Every run consistently reported this output as unsupported", + } + for status, count in analysis["summary"]["output_status_counts"].items(): + lines.append(f"| `{status}` | {count} | {meanings.get(status, '')} |") + + lines.extend( + [ + "", + "## Summary by category", + "", + "| Category | Exact | Numeric variation | Skipped | Inconsistent | Total |", + "|---|---:|---:|---:|---:|---:|", + ] + ) + for category, counts in analysis["summary"]["category_status_counts"].items(): + exact = counts.get("exact", 0) + variable = counts.get("numeric_variation", 0) + skipped = counts.get("skipped_unsupported", 0) + inconsistent = sum( + count + for status, count in counts.items() + if status not in {"exact", "numeric_variation", "skipped_unsupported"} + ) + lines.append( + f"| {markdown_escape(category)} | {exact} | {variable} | {skipped} | {inconsistent} | " + f"{sum(counts.values())} |" + ) + + variable_outputs = [ + output + for output in analysis.get("outputs", []) + if output.get("status") != "exact" + ] + variable_outputs.sort( + key=lambda output: ( + 0 if output.get("status") == "numeric_variation" else 1, + float(output.get("maximum_relative_l2_error") or -1.0), + float(output.get("maximum_absolute_error") or -1.0), + ), + reverse=True, + ) + lines.extend( + [ + "", + "## Variable or inconsistent outputs", + "", + "| Level | Category | Test | Case | Profile | Output | Status | Max relative L2 | Max absolute | Representative |", + "|---|---|---|---|---|---|---|---:|---:|---|", + ] + ) + for output in variable_outputs[:top_output_count]: + identity = output.get("identity", {}) + lines.append( + "| " + + " | ".join( + [ + markdown_escape(output.get("level")), + markdown_escape(output.get("category")), + markdown_escape(identity.get("test_id")), + markdown_escape(identity.get("case_id")), + markdown_escape(identity.get("profile_id")), + markdown_escape(identity.get("output_id")), + f"`{markdown_escape(output.get('status'))}`", + format_number(output.get("maximum_relative_l2_error")), + format_number(output.get("maximum_absolute_error")), + markdown_escape(output.get("representative_run") or "—"), + ] + ) + + " |" + ) + if not variable_outputs: + lines.append("| — | — | — | — | — | — | All outputs exact | — | — | — |") + elif len(variable_outputs) > top_output_count: + lines.append("") + lines.append( + f"The table shows the first {top_output_count} of {len(variable_outputs)} non-exact outputs. " + "The JSON contains all of them" + ) + + lines.extend( + [ + "", + "## Pairwise run overview", + "", + "| Run A | Run B | Structural mismatches | Exact-value mismatches | Max relative L2 | P95 relative L2 |", + "|---|---|---:|---:|---:|---:|", + ] + ) + for pair in analysis.get("environment_pairwise_distances", []): + lines.append( + "| " + + " | ".join( + [ + markdown_escape(pair.get("run_a")), + markdown_escape(pair.get("run_b")), + format_number(pair.get("structural_mismatch_count")), + format_number(pair.get("exact_value_mismatch_count")), + format_number(pair.get("maximum_relative_l2_error")), + format_number(pair.get("p95_relative_l2_error")), + ] + ) + + " |" + ) + + if plot_paths: + lines.extend(["", "## Graphs", ""]) + for plot_path in plot_paths: + lines.append(f"![{plot_path.stem}]({plot_path.name})") + lines.append("") + + lines.extend( + [ + "## Notes for the later GPU comparison", + "", + ( + "The JSON retains every run name, stable output identity, tensor hash, tensor artefact path, " + "pairwise metric and representative-run choice. A later cross-environment tool should use " + "the central numerical policy to assess both this repeatability envelope and the distance " + "between the candidate and reference environments" + ), + "", + ] + ) + path.write_text("\n".join(lines), encoding="utf-8") + + +def make_plots(analysis: Mapping[str, Any], output_directory: Path) -> list[Path]: + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError( + "Matplotlib is required for repeatability graphs. Use --no-plots to skip them" + ) from exc + + def finish_plot(figure: Any, title: str, caption: str) -> None: + figure.suptitle(title, fontsize=12, fontweight="bold", y=0.985) + figure.text( + 0.5, + 0.012, + caption, + ha="center", + va="bottom", + fontsize=8.5, + wrap=True, + ) + figure.tight_layout(rect=(0.0, 0.065, 1.0, 0.94)) + + plot_paths: list[Path] = [] + + categories = analysis["summary"]["category_status_counts"] + if categories: + names = list(categories) + exact = [categories[name].get("exact", 0) for name in names] + variable = [categories[name].get("numeric_variation", 0) for name in names] + skipped = [categories[name].get("skipped_unsupported", 0) for name in names] + inconsistent = [ + sum( + count + for status, count in categories[name].items() + if status not in {"exact", "numeric_variation", "skipped_unsupported"} + ) + for name in names + ] + positions = np.arange(len(names)) + figure, axis = plt.subplots(figsize=(max(9, len(names) * 0.65), 6)) + axis.bar(positions, exact, label="Exact") + axis.bar(positions, variable, bottom=exact, label="Numeric variation") + bottom = np.asarray(exact) + np.asarray(variable) + axis.bar(positions, skipped, bottom=bottom, label="Skipped unsupported") + bottom = bottom + np.asarray(skipped) + axis.bar(positions, inconsistent, bottom=bottom, label="Inconsistent") + axis.set_ylabel("Output count") + axis.set_xticks(positions) + axis.set_xticklabels(names, rotation=55, ha="right") + axis.legend(title="Observed classification") + axis.grid(True, axis="y", alpha=0.25) + finish_plot( + figure, + "Repeatability classification by category", + "Exact outputs match bit-for-bit. Numeric variation is structurally consistent but not " + "identical. Inconsistent includes missing, failed or structurally different outputs.", + ) + path = output_directory / "repeatability_status_by_category.png" + figure.savefig(path, dpi=160) + plt.close(figure) + plot_paths.append(path) + + relative_values = [ + float(value) + for output in analysis.get("outputs", []) + for leaf in output.get("leaves", []) + for pair in leaf.get("pairwise", []) + if isinstance((value := pair.get("relative_l2_error")), (int, float)) + and math.isfinite(float(value)) + and float(value) > 0.0 + ] + if relative_values: + values = np.asarray(relative_values, dtype=np.float64) + low = float(np.min(values)) + high = float(np.max(values)) + bin_count = min(40, max(10, len(relative_values) // 3)) + if high > low: + bins = np.geomspace(low, high, bin_count + 1) + else: + bins = np.geomspace(low / 2.0, high * 2.0, 3) + figure, axis = plt.subplots(figsize=(8, 5)) + axis.hist(values, bins=bins) + axis.set_xscale("log") + axis.set_xlabel("Relative L2 error (log scale)") + axis.set_ylabel("Pairwise leaf comparisons") + axis.grid(True, axis="y", alpha=0.25) + finish_plot( + figure, + "Distribution of non-zero repeatability differences", + "Only non-zero numerical differences are shown. Values further left are smaller and " + "therefore more repeatable; exact matches are excluded from this histogram.", + ) + path = output_directory / "repeatability_relative_l2_distribution.png" + figure.savefig(path, dpi=160) + plt.close(figure) + plot_paths.append(path) + + ranked = [ + output + for output in analysis.get("outputs", []) + if isinstance(output.get("maximum_relative_l2_error"), (int, float)) + and float(output["maximum_relative_l2_error"]) > 0.0 + ] + ranked.sort(key=lambda item: float(item["maximum_relative_l2_error"]), reverse=True) + ranked = ranked[:20] + if ranked: + labels = [ + f"{item['identity']['test_id']}\n{item['identity']['case_id']} / {item['identity']['output_id']}" + for item in reversed(ranked) + ] + values = [float(item["maximum_relative_l2_error"]) for item in reversed(ranked)] + figure, axis = plt.subplots(figsize=(10, max(5, len(ranked) * 0.42))) + positions = np.arange(len(ranked)) + axis.barh(positions, values) + axis.set_xscale("log") + axis.set_yticks(positions) + axis.set_yticklabels(labels) + axis.set_xlabel("Maximum relative L2 error (log scale)") + axis.grid(True, axis="x", alpha=0.25) + finish_plot( + figure, + "Outputs with the largest observed repeatability differences", + "Ranked by the worst pairwise relative L2 error across repeat runs. Shorter bars and " + "values further left indicate stronger repeatability.", + ) + path = output_directory / "repeatability_top_variable_outputs.png" + figure.savefig(path, dpi=160) + plt.close(figure) + plot_paths.append(path) + + return plot_paths + + +def write_json(path: Path, value: Any) -> None: + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def main() -> int: + args = parse_args() + input_directory = args.input_directory.resolve() + output_directory = ( + args.output_directory.resolve() + if args.output_directory + else input_directory / "repeatability_analysis" + ) + output_directory.mkdir(parents=True, exist_ok=True) + + run_directories = discover_run_directories(input_directory, args.pattern) + runs = [load_run_bundle(path) for path in run_directories] + analysis = build_analysis( + runs, + input_root=input_directory, + verify_hash=not args.skip_artifact_hash_check, + ) + + plot_paths = [] if args.no_plots else make_plots(analysis, output_directory) + analysis["plots"] = [path.name for path in plot_paths] + + json_path = output_directory / DEFAULT_JSON_NAME + markdown_path = output_directory / DEFAULT_MARKDOWN_NAME + write_json(json_path, analysis) + write_markdown( + analysis, + markdown_path, + plot_paths=plot_paths, + top_output_count=max(1, args.top_output_count), + ) + + policy_path = None + if args.write_populated_policy is not None: + requested = Path(args.write_populated_policy) + policy_path = requested if requested.is_absolute() else output_directory / requested + template = load_policy_template(args.policy_template.resolve()) + populated = populate_policy_from_repeatability( + template, + analysis, + source_name=json_path.name, + ) + write_policy_json(policy_path, populated) + + print(f"Analysed {len(runs)} repeatability runs") + print(f"Classification: {analysis['overall_classification']}") + print(f"JSON: {json_path}") + print(f"Markdown: {markdown_path}") + for plot_path in plot_paths: + print(f"Graph: {plot_path}") + if policy_path is not None: + print(f"Populated comparison policy: {policy_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/compare_environment_outputs.py b/pytorch/pytorch_extended_tests/manual_comparison_stuff/compare_environment_outputs.py new file mode 100644 index 00000000..afe087cf --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/compare_environment_outputs.py @@ -0,0 +1,874 @@ +#!/usr/bin/env python3 +"""Compare raw result bundles from several environments against ``reference``. + +Expected layout:: + + comparison_root/ + ├── comparison_policy_template.json + ├── reference/ + │ ├── run_001/ + │ └── run_002/ + ├── a100_gcc/ + │ ├── any_run_name/ + │ └── another_run/ + └── h100_clang/ + └── run_001/ + +Each run directory must be an unmodified copy of what the suite wrote to +``/tmp/ci_benchmarks/pytorch``. The script first builds an intra-environment +repeatability analysis for every environment. It then populates the central +policy from the reference runs and compares every candidate run against every +reference run using the raw scalar and tensor values. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np + +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +if str(SCRIPT_DIRECTORY) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIRECTORY)) + +from analyse_repeatability import ( # noqa: E402 + NUMERIC_RECORD_OUTPUT_IDS, + RunBundle, + build_analysis, + canonical_json, + flatten_payload, + load_run_bundle, + load_tensor, +) +from comparison_graphs import make_detailed_model_plots # noqa: E402 +from comparison_policy import ( # noqa: E402 + BUILTIN_TEMPLATE_PATH, + combine_statuses, + compare_analysis_metadata, + judge_numeric_metrics, + judge_repeatability_leaf, + load_policy_template, + populate_policy_from_repeatability, + resolve_leaf_policy, + sha256_json, + write_json, +) + + +COMPARISON_FORMAT_VERSION = "environment_output_comparison_v1" +OUTPUT_DIRECTORY_NAME = "comparison_results" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input_directory", nargs="?", type=Path, default=Path.cwd()) + parser.add_argument("--reference-folder", default="reference") + parser.add_argument("--policy-template", type=Path) + parser.add_argument("--policy-output", type=Path) + parser.add_argument("--output-directory", type=Path) + parser.add_argument("--skip-artifact-hash-check", action="store_true") + parser.add_argument( + "--retain-all-pairs", + action="store_true", + help="Retain every reference/candidate pair in JSON instead of only summaries", + ) + parser.add_argument("--no-plots", action="store_true") + return parser.parse_args() + + +def _is_bundle(path: Path) -> bool: + return all( + (path / name).is_file() + for name in ("run_manifest.json", "observations.jsonl", "test_status.json") + ) + + +def discover_environment_runs(path: Path) -> list[Path]: + if _is_bundle(path): + return [path] + runs = [child for child in sorted(path.iterdir()) if child.is_dir() and _is_bundle(child)] + if not runs: + raise RuntimeError(f"No raw result bundles found beneath environment folder {path}") + return runs + + +def discover_environments( + input_directory: Path, + reference_folder: str, + output_directory: Path, +) -> tuple[Path, list[Path]]: + reference = input_directory / reference_folder + if not reference.is_dir(): + raise RuntimeError(f"Reference environment folder is missing: {reference}") + candidates = [] + for path in sorted(input_directory.iterdir()): + if not path.is_dir() or path == reference or path == output_directory: + continue + try: + discover_environment_runs(path) + except RuntimeError: + continue + candidates.append(path) + if not candidates: + raise RuntimeError(f"No candidate environment folders found beneath {input_directory}") + return reference, candidates + + +def output_map(analysis: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + return { + str(output.get("identity_key")): output + for output in analysis.get("outputs", []) + if isinstance(output, Mapping) + } + + +def leaf_map(output: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + return { + str(leaf.get("path")): leaf + for leaf in output.get("leaves", []) + if isinstance(leaf, Mapping) + } + + +def records_by_key(runs: Sequence[RunBundle]) -> dict[str, dict[str, Mapping[str, Any]]]: + result: dict[str, dict[str, Mapping[str, Any]]] = defaultdict(dict) + for run in runs: + for key, record in run.observations.items(): + result[key][run.run_id] = record + return dict(result) + + +def flattened_produced_record(record: Mapping[str, Any]) -> dict[str, Any]: + if record.get("status") != "produced": + return {} + numeric_scalars = ( + record.get("kind") in {"scalar", "series"} + or record.get("output_id") in NUMERIC_RECORD_OUTPUT_IDS + ) + return { + leaf.path: leaf + for leaf in flatten_payload(record.get("payload"), numeric_scalars=numeric_scalars) + } + + +def _metadata_matches(left: Mapping[str, Any], right: Mapping[str, Any]) -> bool: + return all(left.get(name) == right.get(name) for name in ("logical_dtype", "shape", "numel")) + + +def numeric_array_metrics( + left: np.ndarray, + right: np.ndarray, + limits: Mapping[str, Any], +) -> dict[str, Any]: + if left.shape != right.shape: + return { + "comparable": False, + "reason": "shape_mismatch", + "left_shape": list(left.shape), + "right_shape": list(right.shape), + } + left_values = np.asarray(left) + right_values = np.asarray(right) + left_inexact = np.issubdtype(left_values.dtype, np.inexact) + right_inexact = np.issubdtype(right_values.dtype, np.inexact) + left_nan = np.isnan(left_values) if left_inexact else np.zeros(left.shape, dtype=bool) + right_nan = np.isnan(right_values) if right_inexact else np.zeros(right.shape, dtype=bool) + left_inf = np.isinf(left_values) if left_inexact else np.zeros(left.shape, dtype=bool) + right_inf = np.isinf(right_values) if right_inexact else np.zeros(right.shape, dtype=bool) + left_finite = ~(left_nan | left_inf) + right_finite = ~(right_nan | right_inf) + jointly_finite = left_finite & right_finite + + metrics: dict[str, Any] = { + "comparable": True, + "element_count": int(left_values.size), + "nan_mask_mismatch_count": int(np.count_nonzero(left_nan != right_nan)), + "infinity_mask_mismatch_count": int(np.count_nonzero(left_inf != right_inf)), + "finite_mask_mismatch_count": int(np.count_nonzero(left_finite != right_finite)), + "jointly_finite_count": int(np.count_nonzero(jointly_finite)), + } + if not np.any(jointly_finite): + metrics.update( + { + "exact_equal": bool(np.array_equal(left_values, right_values, equal_nan=True)), + "maximum_absolute_error": 0.0, + "maximum_symmetric_relative_error": 0.0, + "relative_l2_error": 0.0, + "maximum_scaled_error": 0.0, + "bad_count": 0, + "bad_fraction": 0.0, + } + ) + return metrics + + conversion = np.complex128 if np.iscomplexobj(left_values) or np.iscomplexobj(right_values) else np.float64 + left_f = left_values[jointly_finite].astype(conversion) + right_f = right_values[jointly_finite].astype(conversion) + difference = np.abs(left_f - right_f).astype(np.float64) + left_abs = np.abs(left_f).astype(np.float64) + right_abs = np.abs(right_f).astype(np.float64) + scale = np.maximum(left_abs, right_abs) + atol = float(limits.get("atol", 0.0)) + rtol = float(limits.get("rtol", 0.0)) + allowed = atol + rtol * scale + bad = difference > allowed + denominator = np.maximum(scale, np.finfo(np.float64).tiny) + relative = difference / denominator + relative_l2 = float(np.linalg.norm(difference.ravel(), 2)) / max( + float(np.linalg.norm(left_abs.ravel(), 2)), + float(np.linalg.norm(right_abs.ravel(), 2)), + np.finfo(np.float64).tiny, + ) + scaled = difference / np.maximum(allowed, np.finfo(np.float64).tiny) + metrics.update( + { + "exact_equal": bool(np.array_equal(left_values, right_values, equal_nan=True)), + "maximum_absolute_error": float(np.max(difference)), + "maximum_symmetric_relative_error": float(np.max(relative)), + "relative_l2_error": relative_l2, + "maximum_scaled_error": float(np.max(scaled)), + "bad_count": int(np.count_nonzero(bad)), + "bad_fraction": float(np.count_nonzero(bad) / difference.size), + } + ) + return metrics + + +def exact_pair(left: Any, right: Any) -> dict[str, Any]: + matches = canonical_json(left) == canonical_json(right) + return {"status": "PASS" if matches else "FAIL", "exact_equal": matches} + + +def compare_leaf_pair( + reference_run: RunBundle, + reference_leaf: Any, + candidate_run: RunBundle, + candidate_leaf: Any, + leaf_policy: Mapping[str, Any], + family: Mapping[str, Any], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], + tensor_cache: dict[tuple[str, str], np.ndarray], + maybe_multiplier: float, +) -> dict[str, Any]: + if reference_leaf.value_type != candidate_leaf.value_type: + return {"status": "FAIL", "reason": "leaf value types differ"} + if family.get("comparison_type") == "exact": + if reference_leaf.value_type == "tensor": + left = reference_leaf.value + right = candidate_leaf.value + matches = _metadata_matches(left, right) and left.get("sha256") == right.get("sha256") + return { + "status": "PASS" if matches else "FAIL", + "reason": "exact tensor match" if matches else "exact tensor differs", + } + result = exact_pair(reference_leaf.value, candidate_leaf.value) + result["reason"] = "exact values match" if result["status"] == "PASS" else "exact values differ" + return result + + limits = leaf_policy.get("limits", {}).get("cross_environment", {}) + if reference_leaf.value_type == "tensor": + if not _metadata_matches(reference_leaf.value, candidate_leaf.value): + return {"status": "FAIL", "reason": "tensor dtype, shape or element count differs"} + if reference_leaf.value.get("sha256") == candidate_leaf.value.get("sha256"): + metrics = { + "comparable": True, + "exact_equal": True, + "maximum_absolute_error": 0.0, + "maximum_symmetric_relative_error": 0.0, + "relative_l2_error": 0.0, + "maximum_scaled_error": 0.0, + "bad_count": 0, + "bad_fraction": 0.0, + "nan_mask_mismatch_count": 0, + "infinity_mask_mismatch_count": 0, + "finite_mask_mismatch_count": 0, + } + status, ratios, reason = judge_numeric_metrics( + metrics, + limits, + maybe_multiplier=maybe_multiplier, + require_exceptional_masks_match=bool( + family.get("require_exceptional_value_masks_match", True) + ), + ) + return {"status": status, "reason": reason, "metrics": metrics, "limit_ratios": ratios} + left_key = (reference_run.path.as_posix(), str(reference_leaf.value.get("relative_path"))) + right_key = (candidate_run.path.as_posix(), str(candidate_leaf.value.get("relative_path"))) + if left_key not in tensor_cache: + tensor_cache[left_key] = load_tensor( + reference_run, + reference_leaf.value, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if right_key not in tensor_cache: + tensor_cache[right_key] = load_tensor( + candidate_run, + candidate_leaf.value, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + left = tensor_cache[left_key] + right = tensor_cache[right_key] + metrics = numeric_array_metrics(left, right, limits) + elif reference_leaf.value_type == "numeric_scalar": + metrics = numeric_array_metrics( + np.asarray([reference_leaf.value], dtype=np.float64), + np.asarray([candidate_leaf.value], dtype=np.float64), + limits, + ) + else: + return {"status": "FAIL", "reason": "numeric policy was assigned to a non-numeric leaf"} + + status, ratios, reason = judge_numeric_metrics( + metrics, + limits, + maybe_multiplier=maybe_multiplier, + require_exceptional_masks_match=bool( + family.get("require_exceptional_value_masks_match", True) + ), + ) + return {"status": status, "reason": reason, "metrics": metrics, "limit_ratios": ratios} + + +def assess_repeatability_output( + output: Mapping[str, Any] | None, + identity_key: str, + policy: Mapping[str, Any], +) -> dict[str, Any]: + if output is None: + return {"status": "FAIL", "reason": "repeatability output is missing"} + output_policy = policy.get("output_policies", {}).get(identity_key) + if not isinstance(output_policy, Mapping): + return {"status": "NA", "reason": "no populated output policy"} + settings = policy.get("settings", {}) + maybe_multiplier = float(settings.get("maybe_limit_multiplier", 1.5)) + leaves = leaf_map(output) + leaf_results = [] + for path, configured in sorted(output_policy.get("leaves", {}).items()): + leaf = leaves.get(path) + if leaf is None: + leaf_results.append({"path": path, "status": "FAIL", "reason": "leaf is missing"}) + continue + leaf_policy, family = resolve_leaf_policy(policy, identity_key, path) + status, reason, ratios = judge_repeatability_leaf( + leaf, + leaf_policy, + family, + maybe_multiplier=maybe_multiplier, + ) + leaf_results.append({"path": path, "status": status, "reason": reason, "limit_ratios": ratios}) + return { + "status": combine_statuses([str(item["status"]) for item in leaf_results]), + "leaf_results": leaf_results, + } + + +def compare_output_raw( + identity_key: str, + reference_runs: Sequence[RunBundle], + candidate_runs: Sequence[RunBundle], + reference_analysis_output: Mapping[str, Any] | None, + candidate_analysis_output: Mapping[str, Any] | None, + reference_records: Mapping[str, Mapping[str, Any]], + candidate_records: Mapping[str, Mapping[str, Any]], + policy: Mapping[str, Any], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], + tensor_cache: dict[tuple[str, str], np.ndarray], + retain_all_pairs: bool, +) -> dict[str, Any]: + output_policy = policy.get("output_policies", {}).get(identity_key) + identity = ( + reference_analysis_output.get("identity") + if isinstance(reference_analysis_output, Mapping) + else candidate_analysis_output.get("identity") + if isinstance(candidate_analysis_output, Mapping) + else None + ) + base = { + "identity_key": identity_key, + "identity": identity, + "level": reference_analysis_output.get("level") if reference_analysis_output else None, + "category": reference_analysis_output.get("category") if reference_analysis_output else None, + "importance": reference_analysis_output.get("importance") if reference_analysis_output else None, + } + if not isinstance(output_policy, Mapping): + return {**base, "status": "NA", "reason": "no populated output policy"} + if ( + reference_analysis_output is not None + and candidate_analysis_output is not None + and reference_analysis_output.get("kind") != candidate_analysis_output.get("kind") + ): + return {**base, "status": "FAIL", "reason": "output kinds differ"} + + reference_repeatability = assess_repeatability_output(reference_analysis_output, identity_key, policy) + candidate_repeatability = assess_repeatability_output(candidate_analysis_output, identity_key, policy) + if reference_repeatability["status"] == "FAIL": + return { + **base, + "status": "NA", + "reason": "reference repeatability did not qualify", + "reference_repeatability": reference_repeatability, + "candidate_repeatability": candidate_repeatability, + } + reference_paths = set(leaf_map(reference_analysis_output or {})) + candidate_paths = set(leaf_map(candidate_analysis_output or {})) + if reference_paths != candidate_paths: + return { + **base, + "status": "FAIL", + "reason": "reference and candidate output structures contain different leaf paths", + "missing_candidate_leaf_paths": sorted(reference_paths - candidate_paths), + "extra_candidate_leaf_paths": sorted(candidate_paths - reference_paths), + "reference_repeatability": reference_repeatability, + "candidate_repeatability": candidate_repeatability, + } + + settings = policy.get("settings", {}) + maybe_multiplier = float(settings.get("maybe_limit_multiplier", 1.5)) + flattened_reference = { + run.run_id: flattened_produced_record(reference_records.get(run.run_id, {})) + for run in reference_runs + } + flattened_candidate = { + run.run_id: flattened_produced_record(candidate_records.get(run.run_id, {})) + for run in candidate_runs + } + pair_results: list[dict[str, Any]] = [] + leaf_summaries: list[dict[str, Any]] = [] + + for path in sorted(output_policy.get("leaves", {})): + leaf_policy, family = resolve_leaf_policy(policy, identity_key, path) + leaf_pairs: list[dict[str, Any]] = [] + for reference_run in reference_runs: + reference_leaf = flattened_reference.get(reference_run.run_id, {}).get(path) + for candidate_run in candidate_runs: + candidate_leaf = flattened_candidate.get(candidate_run.run_id, {}).get(path) + if reference_leaf is None or candidate_leaf is None: + result = {"status": "FAIL", "reason": "leaf is missing from a raw run"} + else: + result = compare_leaf_pair( + reference_run, + reference_leaf, + candidate_run, + candidate_leaf, + leaf_policy, + family, + verify_hash=verify_hash, + verified_paths=verified_paths, + tensor_cache=tensor_cache, + maybe_multiplier=maybe_multiplier, + ) + pair = { + "path": path, + "reference_run": reference_run.run_id, + "candidate_run": candidate_run.run_id, + **result, + } + pair_results.append(pair) + leaf_pairs.append(pair) + def pair_rank(item: Mapping[str, Any]) -> tuple[int, float]: + status_rank = {"NA": 0, "PASS": 1, "MAYBE": 2, "FAIL": 3}.get(str(item.get("status")), 4) + ratio = max(item.get("limit_ratios", {}).values(), default=0.0) + return status_rank, float(ratio) + + worst_pair = max(leaf_pairs, key=pair_rank) if leaf_pairs else None + leaf_summaries.append( + { + "path": path, + "policy_id": leaf_policy.get("policy_id"), + "status": combine_statuses([str(item["status"]) for item in leaf_pairs]), + "pair_count": len(leaf_pairs), + "status_counts": dict(sorted(Counter(str(item["status"]) for item in leaf_pairs).items())), + "worst_limit_ratio": max( + ( + max(item.get("limit_ratios", {}).values(), default=0.0) + for item in leaf_pairs + ), + default=0.0, + ), + "worst_pair": worst_pair, + } + ) + + cross_status = combine_statuses([str(item["status"]) for item in leaf_summaries]) + status = combine_statuses( + [cross_status, str(candidate_repeatability["status"])] + ) + if len(reference_runs) < int(settings.get("minimum_reference_runs", 3)): + status = combine_statuses([status, "MAYBE"]) + if len(candidate_runs) < int(settings.get("minimum_candidate_runs", 3)): + status = combine_statuses([status, "MAYBE"]) + return { + **base, + "status": status, + "reference_repeatability": reference_repeatability, + "candidate_repeatability": candidate_repeatability, + "cross_environment_status": cross_status, + "leaf_summaries": leaf_summaries, + "pairwise": pair_results if retain_all_pairs else None, + } + + +def compare_environment( + name: str, + reference_runs: Sequence[RunBundle], + candidate_runs: Sequence[RunBundle], + reference_analysis: Mapping[str, Any], + candidate_analysis: Mapping[str, Any], + policy: Mapping[str, Any], + *, + verify_hash: bool, + retain_all_pairs: bool, +) -> dict[str, Any]: + metadata = compare_analysis_metadata(reference_analysis, candidate_analysis) + reference_analysis_outputs = output_map(reference_analysis) + candidate_analysis_outputs = output_map(candidate_analysis) + reference_records_all = records_by_key(reference_runs) + candidate_records_all = records_by_key(candidate_runs) + keys = sorted( + set(policy.get("output_policies", {})) + | set(reference_analysis_outputs) + | set(candidate_analysis_outputs) + ) + verified_paths: set[tuple[str, str]] = set() + tensor_cache: dict[tuple[str, str], np.ndarray] = {} + outputs = [] + for key in keys: + outputs.append( + compare_output_raw( + key, + reference_runs, + candidate_runs, + reference_analysis_outputs.get(key), + candidate_analysis_outputs.get(key), + reference_records_all.get(key, {}), + candidate_records_all.get(key, {}), + policy, + verify_hash=verify_hash, + verified_paths=verified_paths, + tensor_cache=tensor_cache, + retain_all_pairs=retain_all_pairs, + ) + ) + required = [item for item in outputs if item.get("importance") == "required"] + overall = combine_statuses([str(item.get("status")) for item in required or outputs]) + if not metadata["compatible"]: + overall = "FAIL" + counts = Counter(str(item.get("status")) for item in outputs) + level_counts: dict[str, Counter[str]] = defaultdict(Counter) + for output in outputs: + level_counts[str(output.get("level"))][str(output.get("status"))] += 1 + return { + "environment": name, + "overall_status": overall, + "metadata_compatibility": metadata, + "reference_run_count": len(reference_runs), + "candidate_run_count": len(candidate_runs), + "status_counts": dict(sorted(counts.items())), + "level_status_counts": { + level: dict(sorted(values.items())) for level, values in sorted(level_counts.items()) + }, + "verified_tensor_artifact_count": len(verified_paths) if verify_hash else None, + "outputs": outputs, + } + + +def write_markdown( + result: Mapping[str, Any], + path: Path, + plots: Sequence[Path], + plot_details: Sequence[Mapping[str, Any]], +) -> None: + environments = [str(item["environment"]) for item in result.get("environments", [])] + lines = [] + lines.append("| Compared item | " + " | ".join(environments) + " |") + lines.append("|---|" + "---|" * len(environments)) + lines.append( + "| **Overall** | " + + " | ".join(f"**{item['overall_status']}**" for item in result.get("environments", [])) + + " |" + ) + lines.append( + "| Runs | " + + " | ".join(str(item["candidate_run_count"]) for item in result.get("environments", [])) + + " |" + ) + lines.extend(["", "# Environment output comparison details"]) + + levels = sorted( + { + str(output.get("level")) + for environment in result.get("environments", []) + for output in environment.get("outputs", []) + } + ) + for level in levels: + lines.extend(["", f"## {level}", ""]) + lines.append("| Thing compared | " + " | ".join(environments) + " |") + lines.append("|---|" + "---|" * len(environments)) + identities: dict[str, Mapping[str, Any]] = {} + statuses: dict[tuple[str, str], str] = {} + for environment in result.get("environments", []): + for output in environment.get("outputs", []): + if str(output.get("level")) != level: + continue + key = str(output.get("identity_key")) + identities[key] = output.get("identity") or {} + statuses[(str(environment["environment"]), key)] = str(output.get("status")) + for key in sorted(identities): + identity = identities[key] + label = " / ".join( + str(identity.get(name)) + for name in ("test_id", "case_id", "profile_id", "output_id") + ) + lines.append( + "| " + label + " | " + + " | ".join(statuses.get((environment, key), "NA") for environment in environments) + + " |" + ) + + lines.extend( + [ + "", + "## Status meanings", + "", + "- **PASS:** repeatability and all reference-to-candidate raw comparisons are within policy", + "- **MAYBE:** a result is borderline or there are fewer repeats than the policy recommends", + "- **FAIL:** a required exact match, structure check or numerical limit failed", + "- **NA:** there is no valid comparison, commonly because the reference output did not qualify", + ] + ) + if plots: + lines.extend(["", "## Graphs", ""]) + details_by_path = {str(item.get("path")): item for item in plot_details} + for plot in plots: + detail = details_by_path.get(plot.name, {}) + title = str(detail.get("title") or plot.stem) + lines.append(f"### {title}") + lines.append("") + lines.append(f"![{title}]({plot.name})") + caption = detail.get("caption") + if caption: + lines.append("") + lines.append(f"*{caption}*") + lines.append("") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def make_plots(result: Mapping[str, Any], output_directory: Path) -> list[Path]: + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError("Matplotlib is required unless --no-plots is used") from exc + + def finish_plot(figure: Any, title: str, caption: str) -> None: + figure.suptitle(title, fontsize=12, fontweight="bold", y=0.985) + figure.text( + 0.5, + 0.012, + caption, + ha="center", + va="bottom", + fontsize=8.5, + wrap=True, + ) + figure.tight_layout(rect=(0.0, 0.065, 1.0, 0.94)) + + environments = result.get("environments", []) + if not environments: + return [] + names = [str(item["environment"]) for item in environments] + positions = np.arange(len(names)) + bottom = np.zeros(len(names), dtype=np.int64) + figure, axis = plt.subplots(figsize=(max(8, len(names) * 1.3), 5.5)) + for status in ("PASS", "MAYBE", "FAIL", "NA"): + values = np.asarray([item.get("status_counts", {}).get(status, 0) for item in environments]) + axis.bar(positions, values, bottom=bottom, label=status) + bottom += values + axis.set_xticks(positions) + axis.set_xticklabels(names, rotation=35, ha="right") + axis.set_ylabel("Compared output count") + axis.legend(title="Comparison status") + axis.grid(True, axis="y", alpha=0.25) + finish_plot( + figure, + "Cross-environment comparison results", + "PASS is within policy, FAIL exceeds policy or has a structural problem, MAYBE needs raw " + "inspection, and NA could not be compared. Counts include repeatability and cross-environment checks.", + ) + status_path = output_directory / "environment_status_counts.png" + figure.savefig(status_path, dpi=160) + plt.close(figure) + + worst = [] + for environment in environments: + ratios = [ + float(leaf.get("worst_limit_ratio", 0.0)) + for output in environment.get("outputs", []) + for leaf in output.get("leaf_summaries", []) + if math.isfinite(float(leaf.get("worst_limit_ratio", 0.0))) + ] + worst.append(max(ratios, default=0.0)) + positive_logs = [math.log10(value) for value in worst if value > 0.0] + zero_floor = min(-1.0, min(positive_logs, default=0.0) - 1.0) + display_values = [math.log10(value) if value > 0.0 else zero_floor for value in worst] + figure, axis = plt.subplots(figsize=(max(8, len(names) * 1.3), 5)) + bars = axis.bar(positions, display_values) + axis.bar_label(bars, labels=[f"{value:.3g}" for value in worst], padding=3, fontsize=8) + axis.axhline(0.0, linestyle="--", linewidth=1.2, label="Pass limit (ratio = 1)") + axis.set_xticks(positions) + axis.set_xticklabels(names, rotation=35, ha="right") + axis.set_ylabel("log10(worst observed metric ÷ pass limit)") + axis.set_ylim( + min(zero_floor - 0.5, min(display_values, default=zero_floor) - 0.5), + max(1.0, max(display_values, default=0.0) * 1.05), + ) + axis.legend() + axis.grid(True, axis="y", alpha=0.25) + finish_plot( + figure, + "Worst numerical tolerance ratio by environment", + "The dashed zero line is the pass boundary: negative log10 ratios pass and positive ratios " + "exceed a limit. Bar labels show the original ratio; exact zeros are placed at the chart floor.", + ) + ratio_path = output_directory / "environment_worst_tolerance_ratio.png" + figure.savefig(ratio_path, dpi=160) + plt.close(figure) + return [status_path, ratio_path] + + +def main() -> int: + args = parse_args() + input_directory = args.input_directory.resolve() + output_directory = ( + args.output_directory.resolve() + if args.output_directory + else input_directory / OUTPUT_DIRECTORY_NAME + ) + output_directory.mkdir(parents=True, exist_ok=True) + reference_folder, candidate_folders = discover_environments( + input_directory, + args.reference_folder, + output_directory, + ) + + reference_run_paths = discover_environment_runs(reference_folder) + reference_runs = [load_run_bundle(path) for path in reference_run_paths] + runs_by_environment: dict[str, Sequence[RunBundle]] = { + reference_folder.name: reference_runs + } + reference_analysis = build_analysis( + reference_runs, + input_root=reference_folder, + verify_hash=not args.skip_artifact_hash_check, + ) + + if args.policy_template: + template_path = args.policy_template.resolve() + elif (input_directory / "comparison_policy.json").is_file(): + template_path = input_directory / "comparison_policy.json" + elif (input_directory / "comparison_policy_template.json").is_file(): + template_path = input_directory / "comparison_policy_template.json" + else: + template_path = BUILTIN_TEMPLATE_PATH + template = load_policy_template(template_path) + policy = populate_policy_from_repeatability( + template, + reference_analysis, + source_name=reference_folder.name, + ) + policy_output = ( + args.policy_output.resolve() + if args.policy_output + else input_directory / "comparison_policy.json" + ) + write_json(policy_output, policy) + + environment_results = [] + analyses = {reference_folder.name: reference_analysis} + for folder in candidate_folders: + candidate_runs = [load_run_bundle(path) for path in discover_environment_runs(folder)] + runs_by_environment[folder.name] = candidate_runs + candidate_analysis = build_analysis( + candidate_runs, + input_root=folder, + verify_hash=not args.skip_artifact_hash_check, + ) + analyses[folder.name] = candidate_analysis + environment_results.append( + compare_environment( + folder.name, + reference_runs, + candidate_runs, + reference_analysis, + candidate_analysis, + policy, + verify_hash=not args.skip_artifact_hash_check, + retain_all_pairs=args.retain_all_pairs, + ) + ) + + result = { + "comparison_format_version": COMPARISON_FORMAT_VERSION, + "comparison_kind": "raw_cross_environment_output_comparison", + "input_directory": input_directory.as_posix(), + "reference_environment": reference_folder.name, + "reference_run_count": len(reference_runs), + "candidate_environments": [folder.name for folder in candidate_folders], + "policy_file": policy_output.as_posix(), + "policy_sha256": policy.get("policy_sha256") or sha256_json(policy), + "repeatability_summaries": { + name: { + "run_count": analysis.get("run_count"), + "overall_classification": analysis.get("overall_classification"), + "environment_representative_run": analysis.get("environment_representative_run"), + "summary": analysis.get("summary"), + } + for name, analysis in analyses.items() + }, + "retained_all_pair_details": bool(args.retain_all_pairs), + "environments": environment_results, + } + plots: list[Path] = [] + plot_details: list[dict[str, Any]] = [] + if not args.no_plots: + plots.extend(make_plots(result, output_directory)) + plot_details.extend( + make_detailed_model_plots( + runs_by_environment, + analyses, + reference_folder.name, + output_directory, + policy, + verify_hash=not args.skip_artifact_hash_check, + ) + ) + plots.extend(output_directory / item["path"] for item in plot_details) + result["plots"] = [path.name for path in plots] + result["plot_details"] = plot_details + json_path = output_directory / "comparison_results.json" + markdown_path = output_directory / "comparison_results.md" + write_json(json_path, result) + write_markdown(result, markdown_path, plots, plot_details) + + print(f"Reference environment: {reference_folder.name} ({len(reference_runs)} runs)") + print(f"Compared {len(environment_results)} candidate environments") + print(f"Populated policy: {policy_output}") + print(f"JSON: {json_path}") + print(f"Markdown: {markdown_path}") + return 1 if any(item["overall_status"] == "FAIL" for item in environment_results) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/compare_repeatability_analyses.py b/pytorch/pytorch_extended_tests/manual_comparison_stuff/compare_repeatability_analyses.py new file mode 100644 index 00000000..ad9e8013 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/compare_repeatability_analyses.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python3 +"""Compare repeatability-analysis JSON files against ``reference.json``. + +The default input directory is ``repeatability_outputs`` beneath the current +working directory. This script compares the *quality of repeatability* and can +also prove cross-environment equality where the representative values or tensor +hashes match exactly. A changed floating-point tensor hash cannot be judged for +numerical closeness from analysis JSON alone, so that result is marked MAYBE and +left for ``compare_environment_outputs.py``, which reads the raw artefacts. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np + +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +if str(SCRIPT_DIRECTORY) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIRECTORY)) + +from comparison_graphs import make_summary_model_plots # noqa: E402 +from comparison_policy import ( # noqa: E402 + BUILTIN_TEMPLATE_PATH, + combine_statuses, + compare_analysis_metadata, + judge_numeric_metrics, + judge_repeatability_leaf, + load_policy_template, + resolve_leaf_policy, + sha256_json, + write_json, +) + + +RESULT_FORMAT_VERSION = "repeatability_comparison_v1" +DEFAULT_INPUT_DIRECTORY = Path("repeatability_outputs") +DEFAULT_OUTPUT_DIRECTORY_NAME = "repeatability_comparison" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input_directory", + nargs="?", + type=Path, + default=DEFAULT_INPUT_DIRECTORY, + help="Directory containing reference.json and candidate analysis JSON files", + ) + parser.add_argument( + "--policy", + type=Path, + help=( + "Populated comparison policy. Defaults to comparison_policy.json beside " + "this script, then the template if that file does not exist" + ), + ) + parser.add_argument("--output-directory", type=Path) + parser.add_argument("--no-plots", action="store_true") + return parser.parse_args() + + +def read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise RuntimeError(f"Required file is missing: {path}") from exc + except json.JSONDecodeError as exc: + raise RuntimeError(f"File is not valid JSON: {path}") from exc + + +def default_policy_path() -> Path: + populated = SCRIPT_DIRECTORY / "comparison_policy.json" + return populated if populated.is_file() else BUILTIN_TEMPLATE_PATH + + +def discover_analyses(input_directory: Path) -> tuple[Path, list[Path]]: + reference = input_directory / "reference.json" + if not reference.is_file(): + raise RuntimeError(f"Reference analysis is missing: {reference}") + candidates = [] + for path in sorted(input_directory.glob("*.json")): + if not path.is_file() or path.name == "reference.json": + continue + try: + value = read_json(path) + except RuntimeError: + continue + if isinstance(value, Mapping) and value.get("analysis_kind") == "intra_environment_repeatability": + candidates.append(path) + if not candidates: + raise RuntimeError(f"No candidate JSON files found beneath {input_directory}") + return reference, candidates + + +def output_map(analysis: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + return { + str(output.get("identity_key")): output + for output in analysis.get("outputs", []) + if isinstance(output, Mapping) + } + + +def leaf_map(output: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + return { + str(leaf.get("path")): leaf + for leaf in output.get("leaves", []) + if isinstance(leaf, Mapping) + } + + +def representative_value(leaf: Mapping[str, Any]) -> Any: + run = leaf.get("representative_run") + runs = leaf.get("runs") + if isinstance(runs, Mapping) and run in runs: + return runs[run] + return None + + +def scalar_cross_metrics(left: float, right: float, limits: Mapping[str, Any]) -> dict[str, Any]: + left_value = float(left) + right_value = float(right) + absolute = abs(left_value - right_value) + denominator = max(abs(left_value), abs(right_value), np.finfo(np.float64).tiny) + relative = absolute / denominator + threshold = float(limits.get("atol", 0.0)) + float(limits.get("rtol", 0.0)) * denominator + return { + "comparable": True, + "maximum_absolute_error": absolute, + "maximum_symmetric_relative_error": relative, + "relative_l2_error": relative, + "maximum_scaled_error": absolute / max(threshold, np.finfo(np.float64).tiny), + "bad_fraction": 0.0 if absolute <= threshold else 1.0, + "nan_mask_mismatch_count": 0, + "infinity_mask_mismatch_count": 0, + "finite_mask_mismatch_count": 0, + } + + +def exact_value_matches(left: Any, right: Any) -> bool: + return json.dumps(left, sort_keys=True, separators=(",", ":")) == json.dumps( + right, sort_keys=True, separators=(",", ":") + ) + + +def compare_representative_leaf( + reference_leaf: Mapping[str, Any], + candidate_leaf: Mapping[str, Any], + leaf_policy: Mapping[str, Any], + family: Mapping[str, Any], + *, + maybe_multiplier: float, +) -> dict[str, Any]: + reference_value = representative_value(reference_leaf) + candidate_value = representative_value(candidate_leaf) + if reference_value is None or candidate_value is None: + return {"status": "NA", "reason": "representative value is missing"} + + if family.get("comparison_type") == "exact": + matches = exact_value_matches(reference_value, candidate_value) + return { + "status": "PASS" if matches else "FAIL", + "reason": "representative values match exactly" if matches else "exact representative values differ", + } + + if reference_leaf.get("value_type") == "numeric_scalar": + metrics = scalar_cross_metrics( + float(reference_value), + float(candidate_value), + leaf_policy.get("limits", {}).get("cross_environment", {}), + ) + status, ratios, reason = judge_numeric_metrics( + metrics, + leaf_policy.get("limits", {}).get("cross_environment", {}), + maybe_multiplier=maybe_multiplier, + require_exceptional_masks_match=bool( + family.get("require_exceptional_value_masks_match", True) + ), + ) + return {"status": status, "reason": reason, "metrics": metrics, "limit_ratios": ratios} + + if isinstance(reference_value, Mapping) and isinstance(candidate_value, Mapping): + same_metadata = all( + reference_value.get(name) == candidate_value.get(name) + for name in ("logical_dtype", "shape", "numel") + ) + if not same_metadata: + return {"status": "FAIL", "reason": "representative tensor metadata differs"} + if reference_value.get("sha256") == candidate_value.get("sha256"): + return {"status": "PASS", "reason": "representative tensor hashes match exactly"} + return { + "status": "MAYBE", + "reason": "tensor hashes differ and analysis JSON does not contain the raw values", + } + + return {"status": "MAYBE", "reason": "numeric representative cannot be compared from this JSON"} + + +def compare_output( + identity_key: str, + reference_output: Mapping[str, Any] | None, + candidate_output: Mapping[str, Any] | None, + policy: Mapping[str, Any], +) -> dict[str, Any]: + if reference_output is None or candidate_output is None: + return { + "identity_key": identity_key, + "status": "FAIL", + "reason": "output is missing from the reference or candidate analysis", + } + if reference_output.get("kind") != candidate_output.get("kind"): + return { + "identity_key": identity_key, + "identity": reference_output.get("identity"), + "level": reference_output.get("level"), + "category": reference_output.get("category"), + "importance": reference_output.get("importance"), + "status": "FAIL", + "reason": "output kinds differ between reference and candidate", + } + output_policy = policy.get("output_policies", {}).get(identity_key) + if not isinstance(output_policy, Mapping): + return { + "identity_key": identity_key, + "identity": reference_output.get("identity"), + "level": reference_output.get("level"), + "category": reference_output.get("category"), + "importance": reference_output.get("importance"), + "status": "NA", + "reason": "no populated output policy is available", + } + + settings = policy.get("settings", {}) + maybe_multiplier = float(settings.get("maybe_limit_multiplier", 1.5)) + minimum_reference_runs = int(settings.get("minimum_reference_runs", 3)) + minimum_candidate_runs = int(settings.get("minimum_candidate_runs", 3)) + reference_leaves = leaf_map(reference_output) + candidate_leaves = leaf_map(candidate_output) + leaf_results: list[dict[str, Any]] = [] + all_paths = sorted(set(reference_leaves) | set(candidate_leaves) | set(output_policy.get("leaves", {}))) + + for path in all_paths: + reference_leaf = reference_leaves.get(path) + candidate_leaf = candidate_leaves.get(path) + if reference_leaf is None or candidate_leaf is None: + leaf_results.append({"path": path, "status": "FAIL", "reason": "leaf is missing"}) + continue + try: + leaf_policy, family = resolve_leaf_policy(policy, identity_key, path) + except KeyError as exc: + leaf_results.append({"path": path, "status": "NA", "reason": str(exc)}) + continue + ref_status, ref_reason, ref_ratios = judge_repeatability_leaf( + reference_leaf, + leaf_policy, + family, + maybe_multiplier=maybe_multiplier, + ) + cand_status, cand_reason, cand_ratios = judge_repeatability_leaf( + candidate_leaf, + leaf_policy, + family, + maybe_multiplier=maybe_multiplier, + ) + representative = compare_representative_leaf( + reference_leaf, + candidate_leaf, + leaf_policy, + family, + maybe_multiplier=maybe_multiplier, + ) + status = combine_statuses([ref_status, cand_status, str(representative["status"])]) + leaf_results.append( + { + "path": path, + "policy_id": leaf_policy.get("policy_id"), + "status": status, + "reference_repeatability": {"status": ref_status, "reason": ref_reason, "limit_ratios": ref_ratios}, + "candidate_repeatability": {"status": cand_status, "reason": cand_reason, "limit_ratios": cand_ratios}, + "representative_comparison": representative, + } + ) + + statuses = [str(item.get("status")) for item in leaf_results] + status = combine_statuses(statuses) + run_count_note = None + if int(reference_output and len(reference_output.get("available_runs", []))) < minimum_reference_runs: + status = combine_statuses([status, "MAYBE"]) + run_count_note = "reference has fewer runs than the policy recommends" + if int(candidate_output and len(candidate_output.get("available_runs", []))) < minimum_candidate_runs: + status = combine_statuses([status, "MAYBE"]) + run_count_note = "candidate has fewer runs than the policy recommends" + + return { + "identity_key": identity_key, + "identity": reference_output.get("identity"), + "level": reference_output.get("level"), + "category": reference_output.get("category"), + "importance": reference_output.get("importance"), + "status": status, + "run_count_note": run_count_note, + "leaf_results": leaf_results, + } + + +def compare_candidate( + name: str, + reference: Mapping[str, Any], + candidate: Mapping[str, Any], + policy: Mapping[str, Any], +) -> dict[str, Any]: + metadata = compare_analysis_metadata(reference, candidate) + reference_outputs = output_map(reference) + candidate_outputs = output_map(candidate) + keys = sorted(set(reference_outputs) | set(candidate_outputs)) + outputs = [ + compare_output(key, reference_outputs.get(key), candidate_outputs.get(key), policy) + for key in keys + ] + required = [item for item in outputs if item.get("importance") == "required"] + overall = combine_statuses([str(item.get("status")) for item in required or outputs]) + if not metadata["compatible"]: + overall = "FAIL" + counts = Counter(str(item.get("status")) for item in outputs) + by_level: dict[str, Counter[str]] = defaultdict(Counter) + for item in outputs: + by_level[str(item.get("level"))][str(item.get("status"))] += 1 + return { + "candidate": name, + "overall_status": overall, + "metadata_compatibility": metadata, + "status_counts": dict(sorted(counts.items())), + "level_status_counts": { + level: dict(sorted(values.items())) for level, values in sorted(by_level.items()) + }, + "outputs": outputs, + } + + +def format_value(value: Any) -> str: + if value is None: + return "—" + if isinstance(value, float): + return f"{value:.3e}" if value and (abs(value) < 1e-3 or abs(value) >= 1e3) else f"{value:.6g}" + return str(value) + + +def write_markdown( + result: Mapping[str, Any], + path: Path, + plot_paths: Sequence[Path], + plot_details: Sequence[Mapping[str, Any]], +) -> None: + lines = [ + "# Repeatability-analysis comparison", + "", + "| Candidate | Overall | PASS | MAYBE | FAIL | NA |", + "|---|---|---:|---:|---:|---:|", + ] + for candidate in result.get("candidates", []): + counts = candidate.get("status_counts", {}) + lines.append( + f"| {candidate['candidate']} | **{candidate['overall_status']}** | " + f"{counts.get('PASS', 0)} | {counts.get('MAYBE', 0)} | " + f"{counts.get('FAIL', 0)} | {counts.get('NA', 0)} |" + ) + + lines.extend( + [ + "", + "`MAYBE` commonly means that repeatability is acceptable but a representative tensor hash changed. " + "The raw tensor values are needed before numerical closeness can be judged", + ] + ) + + levels = sorted( + { + str(output.get("level")) + for candidate in result.get("candidates", []) + for output in candidate.get("outputs", []) + } + ) + candidate_names = [str(item["candidate"]) for item in result.get("candidates", [])] + for level in levels: + lines.extend(["", f"## {level}", ""]) + headers = " | ".join(["Test / case / profile / output", *candidate_names]) + lines.append(f"| {headers} |") + lines.append("|" + "---|" * (len(candidate_names) + 1)) + identities: dict[str, Mapping[str, Any]] = {} + values: dict[tuple[str, str], str] = {} + for candidate in result.get("candidates", []): + for output in candidate.get("outputs", []): + if str(output.get("level")) != level: + continue + key = str(output.get("identity_key")) + identities[key] = output.get("identity", {}) + values[(str(candidate["candidate"]), key)] = str(output.get("status")) + for key in sorted(identities): + identity = identities[key] + label = " / ".join( + str(identity.get(name)) + for name in ("test_id", "case_id", "profile_id", "output_id") + ) + cells = [label, *[values.get((name, key), "NA") for name in candidate_names]] + lines.append("| " + " | ".join(cells) + " |") + + if plot_paths: + lines.extend(["", "## Graphs", ""]) + details_by_path = {str(item.get("path")): item for item in plot_details} + for plot in plot_paths: + detail = details_by_path.get(plot.name, {}) + title = str(detail.get("title") or plot.stem) + lines.append(f"### {title}") + lines.append("") + lines.append(f"![{title}]({plot.name})") + caption = detail.get("caption") + if caption: + lines.append("") + lines.append(f"*{caption}*") + lines.append("") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def make_plots(result: Mapping[str, Any], output_directory: Path) -> list[Path]: + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError("Matplotlib is required unless --no-plots is used") from exc + + candidates = result.get("candidates", []) + if not candidates: + return [] + names = [str(item["candidate"]) for item in candidates] + statuses = ["PASS", "MAYBE", "FAIL", "NA"] + positions = np.arange(len(names)) + bottom = np.zeros(len(names), dtype=np.int64) + figure, axis = plt.subplots(figsize=(max(8, len(names) * 1.25), 5.5)) + for status in statuses: + values = np.asarray([item.get("status_counts", {}).get(status, 0) for item in candidates]) + axis.bar(positions, values, bottom=bottom, label=status) + bottom += values + axis.set_xticks(positions) + axis.set_xticklabels(names, rotation=35, ha="right") + axis.set_ylabel("Compared output count") + axis.legend(title="Comparison status") + axis.grid(True, axis="y", alpha=0.25) + figure.suptitle( + "Repeatability-analysis comparison status", + fontsize=12, + fontweight="bold", + y=0.985, + ) + figure.text( + 0.5, + 0.012, + "PASS is within policy, FAIL exceeds policy or has a structural problem, MAYBE reflects " + "insufficient summary detail, and NA could not be compared.", + ha="center", + va="bottom", + fontsize=8.5, + wrap=True, + ) + figure.tight_layout(rect=(0.0, 0.065, 1.0, 0.94)) + path = output_directory / "repeatability_comparison_status.png" + figure.savefig(path, dpi=160) + plt.close(figure) + return [path] + + +def main() -> int: + args = parse_args() + input_directory = args.input_directory.resolve() + output_directory = ( + args.output_directory.resolve() + if args.output_directory + else input_directory / DEFAULT_OUTPUT_DIRECTORY_NAME + ) + output_directory.mkdir(parents=True, exist_ok=True) + + reference_path, candidate_paths = discover_analyses(input_directory) + reference = read_json(reference_path) + policy_path = args.policy.resolve() if args.policy else default_policy_path() + policy = load_policy_template(policy_path) + if not policy.get("output_policies"): + raise RuntimeError( + f"The comparison policy is not populated: {policy_path}. " + "Run analyse_repeatability.py with --write-populated-policy first" + ) + + candidate_analyses = {path.stem: read_json(path) for path in candidate_paths} + candidates = [ + compare_candidate(name, reference, analysis, policy) + for name, analysis in candidate_analyses.items() + ] + result = { + "comparison_format_version": RESULT_FORMAT_VERSION, + "comparison_kind": "repeatability_analysis_comparison", + "reference_file": reference_path.name, + "candidate_files": [path.name for path in candidate_paths], + "policy_file": policy_path.as_posix(), + "policy_sha256": policy.get("policy_sha256") or sha256_json(policy), + "candidates": candidates, + } + plot_paths: list[Path] = [] + plot_details: list[dict[str, Any]] = [] + if not args.no_plots: + plot_paths.extend(make_plots(result, output_directory)) + plot_details.extend( + make_summary_model_plots( + reference, + candidate_analyses, + output_directory, + policy, + ) + ) + plot_paths.extend(output_directory / item["path"] for item in plot_details) + result["plots"] = [path.name for path in plot_paths] + result["plot_details"] = plot_details + json_path = output_directory / "repeatability_comparison_results.json" + markdown_path = output_directory / "repeatability_comparison_results.md" + write_json(json_path, result) + write_markdown(result, markdown_path, plot_paths, plot_details) + + print(f"Compared {len(candidates)} repeatability analyses") + print(f"JSON: {json_path}") + print(f"Markdown: {markdown_path}") + return 1 if any(item["overall_status"] == "FAIL" for item in candidates) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_graphs.py b/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_graphs.py new file mode 100644 index 00000000..b47a03d2 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_graphs.py @@ -0,0 +1,1017 @@ +#!/usr/bin/env python3 +"""Plot training progress and inference outputs from comparison inputs.""" + +from __future__ import annotations + +import math +import re +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np + +from analyse_repeatability import RunBundle, load_tensor + + +DEFAULT_REPORTING_SETTINGS = { + "training_progress_checkpoint_count": 5, + "training_loss_step_count": 30, + "include_initial_checkpoint": True, + "inference_preview_value_count": 512, + "inference_scatter_point_count": 2_000, + "inference_sample_error_count": 256, +} + + +def reporting_settings(policy: Mapping[str, Any]) -> dict[str, Any]: + """Return reporting settings with stable defaults for older policy files.""" + + output = dict(DEFAULT_REPORTING_SETTINGS) + configured = policy.get("reporting") + if isinstance(configured, Mapping): + output.update(configured) + return output + + +def _safe_component(value: str) -> str: + cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._") + return cleaned or "unnamed" + + +def _step_number(value: str) -> int | None: + match = re.fullmatch(r"step_(\d+)", value) + return int(match.group(1)) if match else None + + +def _selected_steps(values: Sequence[int], settings: Mapping[str, Any]) -> list[int]: + ordered = sorted(set(int(value) for value in values)) + initial = [value for value in ordered if value == 0] + trained = [value for value in ordered if value > 0] + count = max(1, int(settings.get("training_progress_checkpoint_count", 5))) + selected = trained[:count] + if bool(settings.get("include_initial_checkpoint", True)): + selected = initial[:1] + selected + return selected + + +def _output_map(analysis: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + return { + str(output.get("identity_key")): output + for output in analysis.get("outputs", []) + if isinstance(output, Mapping) + } + + +MODEL_LEVELS = { + "level_0_smoke_workloads", + "level_5_composite_models", + "level_6_real_workloads", +} + + +def _model_outputs(analysis: Mapping[str, Any], output_id: str) -> list[Mapping[str, Any]]: + return [ + output + for output in analysis.get("outputs", []) + if isinstance(output, Mapping) + and output.get("level") in MODEL_LEVELS + and isinstance(output.get("identity"), Mapping) + and output["identity"].get("output_id") == output_id + ] + + +def _plot_filename(prefix: str, identity: Mapping[str, Any], suffix: str) -> str: + parts = [ + prefix, + str(identity.get("test_id")), + str(identity.get("case_id")), + str(identity.get("profile_id")), + suffix, + ] + return "__".join(_safe_component(part) for part in parts) + ".png" + + +def _plot_title(identity: Mapping[str, Any], label: str) -> str: + return ( + f"{identity.get('case_id')} — {label} " + f"[{identity.get('profile_id')}]" + ) + + +def _environment_label(name: str, reference_environment: str = "reference") -> str: + if name != reference_environment: + return name + return "reference baseline" if name == "reference" else f"{name} (reference baseline)" + + +def _reference_axis_label(reference_environment: str) -> str: + if reference_environment == "reference": + return "Reference final logit" + return f"{reference_environment} reference final logit" + + +def _shared_axis_limits(values: np.ndarray) -> tuple[float, float]: + low = float(np.min(values)) + high = float(np.max(values)) + if high > low: + return low, high + padding = max(abs(low) * 0.05, 1.0e-12) + return low - padding, high + padding + + +def _finish_plot(figure: Any, axis: Any, title: str, caption: str) -> None: + figure.suptitle(title, fontsize=12, fontweight="bold", y=0.985) + figure.text( + 0.5, + 0.012, + caption, + ha="center", + va="bottom", + fontsize=8.5, + wrap=True, + ) + figure.tight_layout(rect=(0.0, 0.065, 1.0, 0.94)) + + +def _environment_plot_order(names: Sequence[str], reference_environment: str) -> list[str]: + """Plot candidates first and the reference last so exact overlaps remain visible.""" + + ordered = [name for name in names if name != reference_environment] + if reference_environment in names: + ordered.append(reference_environment) + return ordered + + +def _plot_environment_line( + axis: Any, + x: Sequence[float] | np.ndarray, + y: Sequence[float] | np.ndarray, + name: str, + reference_environment: str, +) -> Any: + """Draw environments with distinct encodings that survive exact curve overlap.""" + + if name == reference_environment: + return axis.plot( + x, + y, + linestyle=(0, (5, 3)), + linewidth=2.3, + marker="o", + markersize=6.0, + markerfacecolor="white", + markeredgewidth=1.5, + label=_environment_label(name, reference_environment), + zorder=4, + )[0] + return axis.plot( + x, + y, + linestyle="-", + linewidth=1.9, + marker="s", + markersize=5.2, + label=_environment_label(name, reference_environment), + zorder=3, + )[0] + + +def _draw_repeatability_envelope( + axis: Any, + x: Sequence[float] | np.ndarray, + lower: Sequence[float] | np.ndarray, + upper: Sequence[float] | np.ndarray, + line: Any, +) -> bool: + """Draw a normal range band, or a faint halo when the range has zero height.""" + + lower_values = np.asarray(lower, dtype=np.float64) + upper_values = np.asarray(upper, dtype=np.float64) + if np.array_equal(lower_values, upper_values, equal_nan=True): + axis.plot( + x, + lower_values, + color=line.get_color(), + linewidth=8.0, + alpha=0.10, + solid_capstyle="round", + zorder=1, + ) + return True + axis.fill_between( + x, + lower_values, + upper_values, + alpha=0.18, + color=line.get_color(), + zorder=1, + ) + return False + + +def _annotate_reference_overlaps( + axis: Any, + plotted_series: Mapping[str, tuple[np.ndarray, np.ndarray]], + reference_environment: str, +) -> list[str]: + """Annotate candidates whose plotted curve is exactly identical to the reference.""" + + reference = plotted_series.get(reference_environment) + if reference is None: + return [] + reference_x, reference_y = reference + matches = [] + for name, (x, y) in plotted_series.items(): + if name == reference_environment: + continue + if ( + np.array_equal(x, reference_x, equal_nan=True) + and np.array_equal(y, reference_y, equal_nan=True) + ): + matches.append(name) + if matches: + reference_label = _environment_label(reference_environment, reference_environment) + axis.text( + 0.02, + 0.97, + "Exact curve overlap\n" + reference_label + " = " + ", ".join(matches), + transform=axis.transAxes, + ha="left", + va="top", + fontsize=8.5, + bbox={"boxstyle": "round,pad=0.35", "facecolor": "white", "alpha": 0.88}, + zorder=6, + ) + return matches + + +def _preview_checkpoint_metrics(output: Mapping[str, Any]) -> dict[int, dict[str, float]]: + preview = output.get("representative_preview") + if not isinstance(preview, Mapping): + return {} + result: dict[int, dict[str, float]] = {} + for step_name, metrics in preview.items(): + step = _step_number(str(step_name)) + if step is None or not isinstance(metrics, Mapping): + continue + values = { + str(name): float(value) + for name, value in metrics.items() + if isinstance(value, (int, float)) and math.isfinite(float(value)) + } + if values: + result[step] = values + return result + + +def _preview_final_logits(output: Mapping[str, Any]) -> Mapping[str, Any] | None: + preview = output.get("representative_preview") + if not isinstance(preview, Mapping): + return None + candidates = [] + for step_name, value in preview.items(): + step = _step_number(str(step_name)) + if step is not None and isinstance(value, Mapping): + logits = value.get("logits") if isinstance(value.get("logits"), Mapping) else value + candidates.append((step, logits)) + return max(candidates, default=(None, None), key=lambda item: -1 if item[0] is None else item[0])[1] + + +def _preview_samples(value: Mapping[str, Any]) -> tuple[np.ndarray, np.ndarray] | None: + indices = value.get("sample_indices") + samples = value.get("sample_values") + if not isinstance(indices, list) or not isinstance(samples, list) or len(indices) != len(samples): + return None + try: + return np.asarray(indices, dtype=np.int64), np.asarray(samples, dtype=np.float64) + except (TypeError, ValueError): + return None + + +def make_summary_model_plots( + reference: Mapping[str, Any], + candidates: Mapping[str, Mapping[str, Any]], + output_directory: Path, + policy: Mapping[str, Any], +) -> list[dict[str, Any]]: + """Plot representative checkpoint metrics and sampled logits from analysis JSON.""" + + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError("Matplotlib is required unless --no-plots is used") from exc + + settings = reporting_settings(policy) + analyses = {"reference": reference, **dict(candidates)} + output_maps = {name: _output_map(analysis) for name, analysis in analyses.items()} + details: list[dict[str, Any]] = [] + + for loss_output_id in ("loss_series", "training_loss"): + for reference_output in _model_outputs(reference, loss_output_id): + key = str(reference_output.get("identity_key")) + identity = reference_output.get("identity") or {} + figure, axis = plt.subplots(figsize=(8.5, 5.2)) + plotted = False + max_steps = max(1, int(settings.get("training_loss_step_count", 30))) + plotted_series: dict[str, tuple[np.ndarray, np.ndarray]] = {} + for name in _environment_plot_order(list(output_maps), "reference"): + outputs = output_maps[name] + output = outputs.get(key) + preview = output.get("representative_preview") if isinstance(output, Mapping) else None + if not isinstance(preview, list): + continue + values = [ + float(value) + for value in preview[:max_steps] + if isinstance(value, (int, float)) and math.isfinite(float(value)) + ] + if not values: + continue + start = 0 if loss_output_id == "loss_series" else 1 + x = np.arange(start, start + len(values)) + y = np.asarray(values, dtype=np.float64) + _plot_environment_line(axis, x, y, name, "reference") + plotted_series[name] = (x, y) + plotted = True + if plotted: + overlaps = _annotate_reference_overlaps(axis, plotted_series, "reference") + title = _plot_title(identity, "Training loss") + caption = ( + "Representative loss preview from each environment. Lower values indicate " + "better fit. Candidates use solid square-marked lines; the reference uses " + "a dashed line with hollow circular markers." + ) + if overlaps: + caption += " The overlap note identifies curves with exactly equal plotted values." + axis.set_xlabel("Optimisation step") + axis.set_ylabel("Training loss") + axis.legend(title="Environment") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + path = output_directory / _plot_filename("summary", identity, "training_loss") + figure.savefig(path, dpi=160) + details.append( + { + "path": path.name, + "kind": "training_step_loss", + "title": title, + "caption": caption, + "identity": identity, + "source": "repeatability_analysis_preview", + } + ) + plt.close(figure) + + for reference_output in _model_outputs(reference, "checkpoint_metrics"): + key = str(reference_output.get("identity_key")) + identity = reference_output.get("identity") or {} + series_by_environment = { + name: _preview_checkpoint_metrics(outputs[key]) + for name, outputs in output_maps.items() + if key in outputs + } + all_steps = sorted({step for series in series_by_environment.values() for step in series}) + steps = _selected_steps(all_steps, settings) + for metric, y_label in (("loss", "Evaluation loss"), ("accuracy", "Evaluation accuracy")): + figure, axis = plt.subplots(figsize=(8.5, 5.2)) + plotted = False + plotted_series: dict[str, tuple[np.ndarray, np.ndarray]] = {} + for name in _environment_plot_order(list(series_by_environment), "reference"): + series = series_by_environment[name] + x = [step for step in steps if metric in series.get(step, {})] + y = [series[step][metric] for step in x] + if not x: + continue + x_values = np.asarray(x, dtype=np.float64) + y_values = np.asarray(y, dtype=np.float64) + _plot_environment_line(axis, x_values, y_values, name, "reference") + plotted_series[name] = (x_values, y_values) + plotted = True + if not plotted: + plt.close(figure) + continue + overlaps = _annotate_reference_overlaps(axis, plotted_series, "reference") + title = _plot_title(identity, y_label) + caption = ( + "Representative evaluation checkpoints from each environment. Candidates use " + "solid square-marked lines; the reference uses a dashed line with hollow circles. " + + ( + "Higher is better; accuracy is shown as a percentage." + if metric == "accuracy" + else "Lower is better." + ) + ) + if overlaps: + caption += " The overlap note identifies curves with exactly equal plotted values." + axis.set_xlabel("Optimisation step at evaluation checkpoint") + axis.set_ylabel(y_label) + if metric == "accuracy": + from matplotlib.ticker import PercentFormatter + + axis.yaxis.set_major_formatter(PercentFormatter(xmax=1.0)) + axis.set_ylim(0.0, 1.0) + axis.legend(title="Environment") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + path = output_directory / _plot_filename("summary", identity, metric) + figure.savefig(path, dpi=160) + plt.close(figure) + details.append( + { + "path": path.name, + "kind": f"training_{metric}", + "title": title, + "caption": caption, + "identity": identity, + "source": "repeatability_analysis_preview", + } + ) + + for inference_output_id in ("evaluation_outputs", "checkpoint_logits"): + for reference_output in _model_outputs(reference, inference_output_id): + key = str(reference_output.get("identity_key")) + identity = reference_output.get("identity") or {} + reference_value = _preview_final_logits(reference_output) + if reference_value is None: + continue + reference_samples = _preview_samples(reference_value) + if reference_samples is None: + continue + reference_indices, reference_values = reference_samples + figure, axis = plt.subplots(figsize=(6.8, 6.2)) + plotted = False + all_values = [reference_values] + for name, outputs in output_maps.items(): + if name == "reference" or key not in outputs: + continue + candidate_value = _preview_final_logits(outputs[key]) + candidate_samples = _preview_samples(candidate_value or {}) + if candidate_samples is None: + continue + candidate_indices, candidate_values = candidate_samples + common, ref_positions, cand_positions = np.intersect1d( + reference_indices, + candidate_indices, + return_indices=True, + ) + if common.size == 0: + continue + maximum = max(1, int(settings.get("inference_preview_value_count", 512))) + if common.size > maximum: + keep = np.linspace(0, common.size - 1, maximum, dtype=np.int64) + ref_positions = ref_positions[keep] + cand_positions = cand_positions[keep] + left = reference_values[ref_positions] + right = candidate_values[cand_positions] + axis.scatter(left, right, s=12, alpha=0.55, label=name) + all_values.append(right) + plotted = True + if not plotted: + plt.close(figure) + continue + finite_parts = [values[np.isfinite(values)] for values in all_values] + finite_parts = [values for values in finite_parts if values.size] + finite_values = np.concatenate(finite_parts) if finite_parts else np.asarray([], dtype=np.float64) + if finite_values.size: + low, high = _shared_axis_limits(finite_values) + axis.plot( + [low, high], + [low, high], + linestyle="--", + linewidth=1.2, + label="Exact agreement (y = x)", + ) + axis.set_xlim(low, high) + axis.set_ylim(low, high) + title = _plot_title(identity, "Sampled final inference logits") + caption = ( + "The reference is encoded on the x-axis. Candidate points on the dashed diagonal " + "match the reference exactly; distance from the diagonal shows logit disagreement." + ) + axis.set_xlabel("Reference sampled logit") + axis.set_ylabel("Candidate sampled logit") + axis.set_aspect("equal", adjustable="box") + axis.legend(title="Candidate / baseline") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + path = output_directory / _plot_filename("summary", identity, "sampled_logits") + figure.savefig(path, dpi=160) + plt.close(figure) + details.append( + { + "path": path.name, + "kind": "inference_sampled_logits", + "title": title, + "caption": caption, + "identity": identity, + "source": "repeatability_analysis_preview", + } + ) + return details + + +def _find_run(runs: Sequence[RunBundle], run_id: str | None) -> RunBundle: + if run_id is not None: + for run in runs: + if run.run_id == run_id: + return run + return sorted(runs, key=lambda item: item.run_id)[0] + + +def _scalar_from_descriptor( + run: RunBundle, + descriptor: Mapping[str, Any], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> float: + values = load_tensor( + run, + descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if values.size != 1: + raise RuntimeError("Expected one scalar tensor in checkpoint_metrics") + return float(np.asarray(values).reshape(-1)[0]) + + +def _raw_training_loss(run: RunBundle, identity_key: str) -> np.ndarray | None: + record = run.observations.get(identity_key) + if not isinstance(record, Mapping) or record.get("status") != "produced": + return None + payload = record.get("payload") + if not isinstance(payload, list): + return None + try: + values = np.asarray(payload, dtype=np.float64) + except (TypeError, ValueError): + return None + return values if values.ndim == 1 else None + + +def _raw_checkpoint_metrics( + run: RunBundle, + identity_key: str, + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> dict[int, dict[str, float]]: + record = run.observations.get(identity_key) + if not isinstance(record, Mapping) or record.get("status") != "produced": + return {} + payload = record.get("payload") + if not isinstance(payload, Mapping): + return {} + result: dict[int, dict[str, float]] = {} + for step_name, metrics in payload.items(): + step = _step_number(str(step_name)) + if step is None or not isinstance(metrics, Mapping): + continue + values = {} + for metric in ("loss", "accuracy"): + descriptor = metrics.get(metric) + if isinstance(descriptor, Mapping) and descriptor.get("artifact_type") == "tensor": + values[metric] = _scalar_from_descriptor( + run, + descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if values: + result[step] = values + return result + + +def _raw_final_logits( + run: RunBundle, + identity_key: str, + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> tuple[int, np.ndarray] | None: + record = run.observations.get(identity_key) + if not isinstance(record, Mapping) or record.get("status") != "produced": + return None + payload = record.get("payload") + if not isinstance(payload, Mapping): + return None + candidates = [] + for step_name, descriptor in payload.items(): + step = _step_number(str(step_name)) + if step is None or not isinstance(descriptor, Mapping): + continue + candidate = descriptor.get("logits") if isinstance(descriptor.get("logits"), Mapping) else descriptor + if candidate.get("artifact_type") == "tensor": + candidates.append((step, candidate)) + if not candidates: + return None + step, descriptor = max(candidates, key=lambda item: item[0]) + return step, load_tensor( + run, + descriptor, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + + +def _common_run_metric_envelope( + runs: Sequence[RunBundle], + identity_key: str, + metric: str, + steps: Sequence[int], + *, + verify_hash: bool, + verified_paths: set[tuple[str, str]], +) -> tuple[np.ndarray, np.ndarray] | None: + rows = [] + for run in runs: + series = _raw_checkpoint_metrics( + run, + identity_key, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if all(metric in series.get(step, {}) for step in steps): + rows.append([series[step][metric] for step in steps]) + if not rows: + return None + values = np.asarray(rows, dtype=np.float64) + return np.min(values, axis=0), np.max(values, axis=0) + + +def make_detailed_model_plots( + runs_by_environment: Mapping[str, Sequence[RunBundle]], + analyses: Mapping[str, Mapping[str, Any]], + reference_environment: str, + output_directory: Path, + policy: Mapping[str, Any], + *, + verify_hash: bool, +) -> list[dict[str, Any]]: + """Plot raw checkpoint metrics and final logits from representative runs.""" + + try: + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError("Matplotlib is required unless --no-plots is used") from exc + + settings = reporting_settings(policy) + reference_analysis = analyses[reference_environment] + output_maps = {name: _output_map(analysis) for name, analysis in analyses.items()} + representative_runs = { + name: _find_run(runs_by_environment[name], analysis.get("environment_representative_run")) + for name, analysis in analyses.items() + } + verified_paths: set[tuple[str, str]] = set() + details: list[dict[str, Any]] = [] + + for loss_output_id in ("loss_series", "training_loss"): + for reference_output in _model_outputs(reference_analysis, loss_output_id): + key = str(reference_output.get("identity_key")) + identity = reference_output.get("identity") or {} + max_steps = max(1, int(settings.get("training_loss_step_count", 30))) + figure, axis = plt.subplots(figsize=(8.5, 5.2)) + plotted = False + plotted_series: dict[str, tuple[np.ndarray, np.ndarray]] = {} + collapsed_bands: list[str] = [] + for name in _environment_plot_order(list(output_maps), reference_environment): + outputs = output_maps[name] + if key not in outputs: + continue + representative = _raw_training_loss(representative_runs[name], key) + if representative is None or representative.size == 0: + continue + count = min(representative.size, max_steps) + start = 0 if loss_output_id == "loss_series" else 1 + x = np.arange(start, start + count) + y = np.asarray(representative[:count], dtype=np.float64) + line = _plot_environment_line(axis, x, y, name, reference_environment) + plotted_series[name] = (x.astype(np.float64), y) + all_runs = [ + values[:count] + for run in runs_by_environment[name] + if (values := _raw_training_loss(run, key)) is not None and values.size >= count + ] + if len(all_runs) > 1: + stacked = np.asarray(all_runs, dtype=np.float64) + if _draw_repeatability_envelope( + axis, + x, + np.min(stacked, axis=0), + np.max(stacked, axis=0), + line, + ): + collapsed_bands.append(name) + plotted = True + if plotted: + overlaps = _annotate_reference_overlaps( + axis, plotted_series, reference_environment + ) + title = _plot_title(identity, "Training loss") + caption = ( + "Each line is the representative run for an environment. Candidates use solid " + "square-marked lines; the reference is drawn last as a dashed line with hollow " + "circles, so exactly coincident curves remain visible. Shaded bands show the " + "minimum-to-maximum range across repeat runs; lower loss is better." + ) + if collapsed_bands: + caption += ( + " For " + ", ".join(collapsed_bands) + + ", all plotted repeats are identical, so the zero-width band is shown " + "as a faint halo around the line." + ) + if overlaps: + caption += " The in-plot overlap note confirms exact equality at every plotted step." + axis.set_xlabel("Optimisation step") + axis.set_ylabel("Training loss") + axis.legend(title="Environment") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + path = output_directory / _plot_filename("detailed", identity, "training_loss") + figure.savefig(path, dpi=160) + details.append( + { + "path": path.name, + "kind": "training_step_loss", + "title": title, + "caption": caption, + "identity": identity, + "source": "raw_representative_runs_with_repeatability_envelope", + } + ) + plt.close(figure) + + for reference_output in _model_outputs(reference_analysis, "checkpoint_metrics"): + key = str(reference_output.get("identity_key")) + identity = reference_output.get("identity") or {} + representative_series = { + name: _raw_checkpoint_metrics( + representative_runs[name], + key, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + for name, outputs in output_maps.items() + if key in outputs + } + all_steps = sorted({step for series in representative_series.values() for step in series}) + steps = _selected_steps(all_steps, settings) + for metric, y_label in (("loss", "Evaluation loss"), ("accuracy", "Evaluation accuracy")): + figure, axis = plt.subplots(figsize=(8.5, 5.2)) + plotted = False + plotted_series: dict[str, tuple[np.ndarray, np.ndarray]] = {} + collapsed_bands: list[str] = [] + for name in _environment_plot_order(list(representative_series), reference_environment): + series = representative_series[name] + x = [step for step in steps if metric in series.get(step, {})] + y = [series[step][metric] for step in x] + if not x: + continue + x_values = np.asarray(x, dtype=np.float64) + y_values = np.asarray(y, dtype=np.float64) + line = _plot_environment_line( + axis, x_values, y_values, name, reference_environment + ) + plotted_series[name] = (x_values, y_values) + envelope = _common_run_metric_envelope( + runs_by_environment[name], + key, + metric, + x, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if envelope is not None and len(runs_by_environment[name]) > 1: + lower, upper = envelope + if _draw_repeatability_envelope(axis, x_values, lower, upper, line): + collapsed_bands.append(name) + plotted = True + if not plotted: + plt.close(figure) + continue + overlaps = _annotate_reference_overlaps( + axis, plotted_series, reference_environment + ) + title = _plot_title(identity, y_label) + caption = ( + "Each line is the representative run for an environment. Candidates use solid " + "square-marked lines; the reference uses a dashed line with hollow circles. " + "Shaded bands show the minimum-to-maximum range across repeat runs. " + + ( + "Higher is better; accuracy is shown as a percentage." + if metric == "accuracy" + else "Lower is better." + ) + ) + if collapsed_bands: + caption += ( + " For " + ", ".join(collapsed_bands) + + ", the repeat-run range is exactly zero and is shown as a faint halo." + ) + if overlaps: + caption += " The in-plot overlap note confirms exact equality at every checkpoint." + axis.set_xlabel("Optimisation step at evaluation checkpoint") + axis.set_ylabel(y_label) + if metric == "accuracy": + from matplotlib.ticker import PercentFormatter + + axis.yaxis.set_major_formatter(PercentFormatter(xmax=1.0)) + axis.set_ylim(0.0, 1.0) + axis.legend(title="Environment") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + path = output_directory / _plot_filename("detailed", identity, metric) + figure.savefig(path, dpi=160) + plt.close(figure) + details.append( + { + "path": path.name, + "kind": f"training_{metric}", + "title": title, + "caption": caption, + "identity": identity, + "source": "raw_representative_runs_with_repeatability_envelope", + } + ) + + for inference_output_id in ("evaluation_outputs", "checkpoint_logits"): + for reference_output in _model_outputs(reference_analysis, inference_output_id): + key = str(reference_output.get("identity_key")) + identity = reference_output.get("identity") or {} + reference_result = _raw_final_logits( + representative_runs[reference_environment], + key, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if reference_result is None: + continue + reference_step, reference_logits = reference_result + candidate_logits = {} + for name, outputs in output_maps.items(): + if name == reference_environment or key not in outputs: + continue + result = _raw_final_logits( + representative_runs[name], + key, + verify_hash=verify_hash, + verified_paths=verified_paths, + ) + if result is None or result[0] != reference_step or result[1].shape != reference_logits.shape: + continue + candidate_logits[name] = result[1] + if not candidate_logits: + continue + + flat_reference = np.asarray(reference_logits).reshape(-1).astype(np.float64) + point_count = min( + flat_reference.size, + max(1, int(settings.get("inference_scatter_point_count", 2_000))), + ) + indices = np.linspace(0, flat_reference.size - 1, point_count, dtype=np.int64) + figure, axis = plt.subplots(figsize=(6.8, 6.2)) + all_values = [flat_reference[indices]] + for name, logits in candidate_logits.items(): + values = np.asarray(logits).reshape(-1).astype(np.float64)[indices] + axis.scatter(flat_reference[indices], values, s=10, alpha=0.5, label=name) + all_values.append(values) + finite_parts = [values[np.isfinite(values)] for values in all_values] + finite_parts = [values for values in finite_parts if values.size] + finite_values = np.concatenate(finite_parts) if finite_parts else np.asarray([], dtype=np.float64) + if finite_values.size: + low, high = _shared_axis_limits(finite_values) + axis.plot( + [low, high], + [low, high], + linestyle="--", + linewidth=1.2, + label="Exact agreement (y = x)", + ) + axis.set_xlim(low, high) + axis.set_ylim(low, high) + title = _plot_title(identity, f"Final inference logits at step {reference_step}") + caption = ( + f"The {reference_environment} representative run is encoded on the x-axis. " + "Candidate points on the dashed diagonal match it exactly; distance from the " + "diagonal shows logit disagreement." + ) + axis.set_xlabel(_reference_axis_label(reference_environment)) + axis.set_ylabel("Candidate final logit") + axis.set_aspect("equal", adjustable="box") + axis.legend(title="Candidate / baseline") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + scatter_path = output_directory / _plot_filename("detailed", identity, "final_logits_scatter") + figure.savefig(scatter_path, dpi=160) + plt.close(figure) + details.append( + { + "path": scatter_path.name, + "kind": "inference_logits_scatter", + "title": title, + "caption": caption, + "identity": identity, + "source": "raw_representative_runs", + } + ) + + sample_errors: dict[str, np.ndarray] = {} + reference_array = np.asarray(reference_logits, dtype=np.float64) + for name, logits in candidate_logits.items(): + difference = np.abs(np.asarray(logits, dtype=np.float64) - reference_array) + if difference.ndim >= 2: + errors = np.max(difference.reshape(difference.shape[0], -1), axis=1) + else: + errors = difference.reshape(-1) + sample_errors[name] = errors + max_samples = max(1, int(settings.get("inference_sample_error_count", 256))) + figure, axis = plt.subplots(figsize=(9, 5.2)) + axis.axhline( + 0.0, + linestyle="--", + linewidth=1.2, + label=f"{reference_environment} baseline (zero error)", + ) + for name, errors in sample_errors.items(): + count = min(errors.size, max_samples) + sample_indices = np.linspace(0, errors.size - 1, count, dtype=np.int64) + axis.plot(sample_indices, errors[sample_indices], label=name) + title = _plot_title(identity, "Per-sample final inference error") + caption = ( + f"Each candidate curve is |candidate − {reference_environment}| for the largest " + "logit error in each sampled evaluation item. The dashed zero line is the " + "reference self-comparison baseline; lower is better." + ) + axis.set_xlabel("Evaluation sample index") + axis.set_ylabel(f"Maximum absolute logit error from {reference_environment}") + axis.set_ylim(bottom=0.0) + axis.legend(title="Environment / baseline") + axis.grid(True, alpha=0.25) + _finish_plot(figure, axis, title, caption) + error_path = output_directory / _plot_filename("detailed", identity, "final_logits_error") + figure.savefig(error_path, dpi=160) + plt.close(figure) + details.append( + { + "path": error_path.name, + "kind": "inference_per_sample_error", + "title": title, + "caption": caption, + "identity": identity, + "source": "raw_representative_runs", + } + ) + + if reference_array.ndim >= 2 and reference_array.shape[-1] > 1: + reference_predictions = np.argmax(reference_array, axis=-1).reshape(-1) + names = [_environment_label(reference_environment, reference_environment)] + disagreement_fractions = [0.0] + for name, logits in candidate_logits.items(): + predictions = np.argmax(np.asarray(logits), axis=-1).reshape(-1) + names.append(name) + disagreement_fractions.append( + float(np.mean(predictions != reference_predictions)) + ) + figure, axis = plt.subplots(figsize=(max(7, len(names) * 1.3), 4.8)) + positions = np.arange(len(names)) + bars = axis.bar(positions, disagreement_fractions) + axis.bar_label( + bars, + labels=[f"{value:.2%}" for value in disagreement_fractions], + padding=3, + fontsize=8, + ) + title = _plot_title(identity, "Final prediction disagreements") + caption = ( + f"Fraction of evaluation items whose predicted class differs from the " + f"{reference_environment} representative run. The reference self-comparison " + "is included explicitly at 0%; lower is better." + ) + axis.set_xticks(positions) + axis.set_xticklabels(names, rotation=35, ha="right") + axis.set_ylabel(f"Prediction disagreement from {reference_environment}") + from matplotlib.ticker import PercentFormatter + + axis.yaxis.set_major_formatter(PercentFormatter(xmax=1.0)) + maximum_disagreement = max(disagreement_fractions, default=0.0) + axis.set_ylim(0.0, max(0.01, min(1.0, maximum_disagreement * 1.25 + 0.005))) + axis.axhline(0.0, linestyle="--", linewidth=1.0) + axis.grid(True, axis="y", alpha=0.25) + _finish_plot(figure, axis, title, caption) + disagreement_path = output_directory / _plot_filename( + "detailed", identity, "prediction_disagreements" + ) + figure.savefig(disagreement_path, dpi=160) + plt.close(figure) + details.append( + { + "path": disagreement_path.name, + "kind": "inference_prediction_disagreement", + "title": title, + "caption": caption, + "identity": identity, + "source": "raw_representative_runs", + } + ) + return details diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_policy.py b/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_policy.py new file mode 100644 index 00000000..06ed2924 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_policy.py @@ -0,0 +1,422 @@ +"""Populate and apply the central comparison policy. + +This module is shared by the repeatability analyser and the two comparison +scripts. Keeping the calibration rules here means the same limits are used when +we compare analysis JSON and when we compare the underlying tensor artefacts. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import math +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, MutableMapping, Sequence + + +POLICY_FORMAT_VERSION = "comparison_policy_v1" +BUILTIN_TEMPLATE_PATH = Path(__file__).with_name("comparison_policy_template.json") +STATUS_ORDER = {"NA": 0, "PASS": 1, "MAYBE": 2, "FAIL": 3} + +COMPARABLE_ANALYSIS_FIELDS = ( + "suite_name", + "suite_version", + "result_format_version", + "test_catalogue_version", + "root_seed", + "dataset_manifest_sha256", + "profile_ids", +) + + +def analysis_metadata(analysis: Mapping[str, Any]) -> dict[str, Any]: + fields = analysis.get("compatibility", {}).get("fields", {}) + return { + name: fields.get(name, {}).get("common_value") + for name in COMPARABLE_ANALYSIS_FIELDS + } + + +def compare_analysis_metadata( + reference: Mapping[str, Any], + candidate: Mapping[str, Any], +) -> dict[str, Any]: + reference_values = analysis_metadata(reference) + candidate_values = analysis_metadata(candidate) + mismatches = [ + name + for name in COMPARABLE_ANALYSIS_FIELDS + if canonical_json(reference_values.get(name)) != canonical_json(candidate_values.get(name)) + ] + return { + "compatible": not mismatches, + "mismatched_fields": mismatches, + "reference": reference_values, + "candidate": candidate_values, + } + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def sha256_json(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise RuntimeError(f"Required JSON file is missing: {path}") from exc + except json.JSONDecodeError as exc: + raise RuntimeError(f"File is not valid JSON: {path}") from exc + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def _fill_missing(target: MutableMapping[str, Any], defaults: Mapping[str, Any]) -> None: + """Add absent values without replacing deliberate local edits.""" + + for key, value in defaults.items(): + if key not in target or target[key] is None: + target[key] = copy.deepcopy(value) + elif isinstance(target[key], MutableMapping) and isinstance(value, Mapping): + _fill_missing(target[key], value) + + +def load_policy_template(path: Path | None = None) -> dict[str, Any]: + template_path = path or BUILTIN_TEMPLATE_PATH + loaded = read_json(template_path) + if not isinstance(loaded, dict): + raise RuntimeError(f"Comparison policy must be a JSON object: {template_path}") + + # Add newly introduced built-in policies to older templates + # Existing non-null values are always kept + builtin = read_json(BUILTIN_TEMPLATE_PATH) + _fill_missing(loaded, builtin) + if loaded.get("comparison_policy_format_version") != POLICY_FORMAT_VERSION: + raise RuntimeError( + "Unsupported comparison policy format: " + f"{loaded.get('comparison_policy_format_version')!r}" + ) + return loaded + + +def _normalise_dtype(value: str | None) -> str: + if not value: + return "unknown" + return value.lower().replace("torch.", "").replace("numpy.", "") + + +def _tensor_dtype(leaf: Mapping[str, Any]) -> str: + representative = leaf.get("representative_run") + runs = leaf.get("runs") + if not isinstance(runs, Mapping): + return "unknown" + descriptor = runs.get(representative) if representative in runs else next(iter(runs.values()), None) + if isinstance(descriptor, Mapping): + return _normalise_dtype(str(descriptor.get("logical_dtype") or "unknown")) + return "unknown" + + +def select_policy_id(leaf: Mapping[str, Any]) -> str: + value_type = str(leaf.get("value_type")) + if value_type not in {"tensor", "numeric_scalar"}: + return "exact.value.v1" + if value_type == "numeric_scalar": + return "numeric.scalar.v1" + + dtype = _tensor_dtype(leaf) + if dtype in {"bool", "uint8", "uint16", "uint32", "uint64", "int8", "int16", "int32", "int64"}: + return "exact.value.v1" + if dtype in {"float64", "complex128"}: + return "numeric.float64.v1" + if dtype in {"float32", "complex64"}: + return "numeric.float32.v1" + if dtype == "float16": + return "numeric.float16.v1" + if dtype == "bfloat16": + return "numeric.bfloat16.v1" + return "numeric.fallback.v1" + + +def _finite_number(value: Any, default: float = 0.0) -> float: + if isinstance(value, (int, float)) and math.isfinite(float(value)): + return float(value) + return default + + +def _calibrated_limit(base: float, observed: float, multiplier: float, hard: float) -> tuple[float, bool]: + requested = max(base, observed * multiplier) + return min(requested, hard), requested > hard + + +def _policy_limits( + policy: Mapping[str, Any], + stage: str, + observed: Mapping[str, Any], + multiplier: float, +) -> tuple[dict[str, float], bool]: + base = policy.get("base_limits", {}).get(stage, {}) + hard = policy.get("hard_limits", {}) + observed_values = { + "atol": _finite_number(observed.get("maximum_absolute_error")), + "rtol": _finite_number(observed.get("maximum_symmetric_relative_error")), + "relative_l2": _finite_number(observed.get("maximum_relative_l2_error")), + "maximum_bad_fraction": 0.0, + } + result: dict[str, float] = {} + exceeded = False + for name in ("atol", "rtol", "relative_l2", "maximum_bad_fraction"): + value, clipped = _calibrated_limit( + _finite_number(base.get(name)), + observed_values[name], + multiplier, + _finite_number(hard.get(name), float("inf")), + ) + result[name] = value + exceeded = exceeded or clipped + return result, exceeded + + +def _reference_leaf_envelope(leaf: Mapping[str, Any]) -> dict[str, Any]: + summary = leaf.get("summary") if isinstance(leaf.get("summary"), Mapping) else {} + maximum_symmetric = 0.0 + for pair in leaf.get("pairwise", []): + maximum_symmetric = max( + maximum_symmetric, + _finite_number(pair.get("maximum_symmetric_relative_error")), + ) + return { + "status": leaf.get("status"), + "maximum_absolute_error": summary.get("maximum_absolute_error"), + "maximum_relative_l2_error": summary.get("maximum_relative_l2_error"), + "maximum_symmetric_relative_error": maximum_symmetric, + "maximum_mismatch_fraction": summary.get("maximum_mismatch_fraction"), + } + + +def populate_policy_from_repeatability( + template: Mapping[str, Any], + analysis: Mapping[str, Any], + *, + source_name: str | None = None, +) -> dict[str, Any]: + """Return a policy populated from one reference repeatability analysis. + + Existing values in ``output_policies`` are preserved. Missing fields and new + output identities are added, which lets a manually tuned policy survive a + later recalibration run + """ + + policy = copy.deepcopy(dict(template)) + _fill_missing(policy, read_json(BUILTIN_TEMPLATE_PATH)) + settings = policy["settings"] + multiplier = float(settings["reference_variability_multiplier"]) + output_policies = policy.setdefault("output_policies", {}) + + unstable_outputs: list[str] = [] + for output in analysis.get("outputs", []): + identity_key = str(output.get("identity_key")) + generated = { + "identity": copy.deepcopy(output.get("identity")), + "level": output.get("level"), + "category": output.get("category"), + "kind": output.get("kind"), + "importance": output.get("importance"), + "reference_status": output.get("status"), + "leaves": {}, + } + reference_output_unstable = str(output.get("status")) not in {"exact", "numeric_variation"} + for leaf in output.get("leaves", []): + leaf_path = str(leaf.get("path")) + policy_id = select_policy_id(leaf) + family = policy["policies"][policy_id] + leaf_entry: dict[str, Any] = { + "value_type": leaf.get("value_type"), + "policy_id": policy_id, + "reference_envelope": _reference_leaf_envelope(leaf), + } + if family.get("comparison_type") == "exact": + leaf_entry["limits"] = {"repeatability": {}, "cross_environment": {}} + leaf_entry["reference_qualified"] = leaf.get("status") == "exact" + else: + observed = leaf_entry["reference_envelope"] + repeat_limits, repeat_clipped = _policy_limits( + family, "repeatability", observed, multiplier + ) + cross_limits, cross_clipped = _policy_limits( + family, "cross_environment", observed, multiplier + ) + leaf_entry["limits"] = { + "repeatability": repeat_limits, + "cross_environment": cross_limits, + } + leaf_entry["reference_qualified"] = ( + leaf.get("status") in {"exact", "numeric_variation"} + and not repeat_clipped + and not cross_clipped + ) + leaf_entry["hard_limit_was_reached"] = repeat_clipped or cross_clipped + reference_output_unstable = reference_output_unstable or not leaf_entry["reference_qualified"] + generated["leaves"][leaf_path] = leaf_entry + + generated["reference_qualified"] = not reference_output_unstable + if reference_output_unstable: + unstable_outputs.append(identity_key) + existing = output_policies.get(identity_key) + if isinstance(existing, Mapping) and existing.get("lock_calibration") is True: + # Keep a deliberate output-specific policy exactly as supplied + # Family-level hardcoded values are already preserved separately + continue + output_policies[identity_key] = generated + + policy["calibration"] = { + **dict(policy.get("calibration", {})), + "generated_at_utc": utc_now(), + "source_name": source_name, + "source_analysis_format_version": analysis.get("analysis_format_version"), + "source_analysis_sha256": sha256_json(analysis), + "reference_run_count": analysis.get("run_count"), + "reference_overall_classification": analysis.get("overall_classification"), + "output_policy_count": len(output_policies), + "reference_unstable_output_count": len(unstable_outputs), + "reference_unstable_output_keys": sorted(unstable_outputs), + } + semantic_policy = { + key: value + for key, value in policy.items() + if key not in {"policy_sha256", "calibration"} + } + policy["policy_sha256"] = sha256_json(semantic_policy) + return policy + + +def resolve_leaf_policy( + policy: Mapping[str, Any], + identity_key: str, + leaf_path: str, +) -> tuple[Mapping[str, Any], Mapping[str, Any]]: + output = policy.get("output_policies", {}).get(identity_key) + if not isinstance(output, Mapping): + raise KeyError(f"No output policy for {identity_key}") + leaf = output.get("leaves", {}).get(leaf_path) + if not isinstance(leaf, Mapping): + raise KeyError(f"No leaf policy for {identity_key} {leaf_path}") + family = policy.get("policies", {}).get(leaf.get("policy_id")) + if not isinstance(family, Mapping): + raise KeyError(f"Unknown policy family {leaf.get('policy_id')!r}") + return leaf, family + + +def combine_statuses(statuses: Sequence[str]) -> str: + if not statuses: + return "NA" + return max(statuses, key=lambda value: STATUS_ORDER.get(value, 99)) + + +def numeric_limit_ratios(metrics: Mapping[str, Any], limits: Mapping[str, Any]) -> dict[str, float]: + # Raw comparisons provide a scaled allclose-style error and a bad-element fraction + # Repeatability JSON only has aggregate absolute and symmetric-relative envelopes + if isinstance(metrics.get("maximum_scaled_error"), (int, float)): + mapping = { + "maximum_scaled_error": None, + "relative_l2_error": "relative_l2", + "bad_fraction": "maximum_bad_fraction", + } + else: + mapping = { + "maximum_absolute_error": "atol", + "maximum_symmetric_relative_error": "rtol", + "relative_l2_error": "relative_l2", + "bad_fraction": "maximum_bad_fraction", + } + ratios: dict[str, float] = {} + for metric_name, limit_name in mapping.items(): + metric = _finite_number(metrics.get(metric_name)) + limit = 1.0 if limit_name is None else _finite_number(limits.get(limit_name)) + if limit == 0.0: + ratios[metric_name] = 0.0 if metric == 0.0 else 1e300 + else: + ratios[metric_name] = metric / limit + return ratios + + +def judge_numeric_metrics( + metrics: Mapping[str, Any], + limits: Mapping[str, Any], + *, + maybe_multiplier: float, + require_exceptional_masks_match: bool = True, +) -> tuple[str, dict[str, float], str]: + if not metrics.get("comparable", True): + return "FAIL", {}, str(metrics.get("reason") or "not comparable") + if require_exceptional_masks_match and any( + int(metrics.get(name, 0)) > 0 + for name in ( + "nan_mask_mismatch_count", + "infinity_mask_mismatch_count", + "finite_mask_mismatch_count", + ) + ): + return "FAIL", {}, "NaN, infinity or finite-value masks differ" + + ratios = numeric_limit_ratios(metrics, limits) + worst = max(ratios.values(), default=0.0) + if worst <= 1.0: + return "PASS", ratios, "within the populated numerical limits" + if worst <= maybe_multiplier: + return "MAYBE", ratios, "outside the pass limit but still within the review band" + return "FAIL", ratios, "outside the populated numerical limits" + + +def judge_repeatability_leaf( + leaf: Mapping[str, Any], + leaf_policy: Mapping[str, Any], + family: Mapping[str, Any], + *, + maybe_multiplier: float, +) -> tuple[str, str, dict[str, float]]: + status = str(leaf.get("status")) + if family.get("comparison_type") == "exact": + if status == "exact": + return "PASS", "all repeat runs match exactly", {} + return "FAIL", f"exact policy but repeatability status is {status}", {} + if status not in {"exact", "numeric_variation"}: + return "FAIL", f"repeatability status is {status}", {} + + envelope = _reference_leaf_envelope(leaf) + metrics = { + "comparable": True, + "maximum_absolute_error": envelope.get("maximum_absolute_error"), + "maximum_symmetric_relative_error": envelope.get("maximum_symmetric_relative_error"), + "relative_l2_error": envelope.get("maximum_relative_l2_error"), + "bad_fraction": 0.0, + "nan_mask_mismatch_count": 0, + "infinity_mask_mismatch_count": 0, + "finite_mask_mismatch_count": 0, + } + return_value, ratios, reason = judge_numeric_metrics( + metrics, + leaf_policy.get("limits", {}).get("repeatability", {}), + maybe_multiplier=maybe_multiplier, + require_exceptional_masks_match=bool( + family.get("require_exceptional_value_masks_match", True) + ), + ) + return return_value, reason, ratios diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_policy_template.json b/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_policy_template.json new file mode 100644 index 00000000..ea0517a6 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/comparison_policy_template.json @@ -0,0 +1,246 @@ +{ + "comparison_policy_format_version": "comparison_policy_v1", + "policy_version": "v1", + "description": "Central numerical comparison policy. Static floors and hard ceilings are deliberate defaults. Per-output calibrated limits are added from reference repeatability without replacing existing values.", + "settings": { + "minimum_reference_runs": 3, + "minimum_candidate_runs": 3, + "reference_variability_multiplier": 3.0, + "maybe_limit_multiplier": 1.5, + "near_limit_fraction": 0.75, + "require_matching_suite_metadata": true + }, + "policies": { + "exact.value.v1": { + "comparison_type": "exact", + "description": "Shapes, dtypes, indices, booleans, strings, integer tensors and other values which must match exactly", + "locked_fields": [ + "comparison_type", + "require_exact", + "require_shape_match", + "require_dtype_match", + "require_exceptional_value_masks_match" + ], + "require_exact": true, + "require_shape_match": true, + "require_dtype_match": true, + "require_exceptional_value_masks_match": true + }, + "numeric.float64.v1": { + "comparison_type": "numeric", + "description": "Ordinary FP64 and complex128 values", + "locked_fields": [ + "comparison_type", + "base_limits", + "hard_limits", + "require_shape_match", + "require_dtype_match", + "require_exceptional_value_masks_match" + ], + "require_shape_match": true, + "require_dtype_match": true, + "require_exceptional_value_masks_match": true, + "base_limits": { + "repeatability": { + "atol": 1e-13, + "rtol": 1e-11, + "relative_l2": 1e-11, + "maximum_bad_fraction": 0.0 + }, + "cross_environment": { + "atol": 1e-12, + "rtol": 1e-10, + "relative_l2": 1e-10, + "maximum_bad_fraction": 0.0 + } + }, + "hard_limits": { + "atol": 1e-09, + "rtol": 1e-07, + "relative_l2": 1e-07, + "maximum_bad_fraction": 0.001 + } + }, + "numeric.float32.v1": { + "comparison_type": "numeric", + "description": "Ordinary FP32 and complex64 values", + "locked_fields": [ + "comparison_type", + "base_limits", + "hard_limits", + "require_shape_match", + "require_dtype_match", + "require_exceptional_value_masks_match" + ], + "require_shape_match": true, + "require_dtype_match": true, + "require_exceptional_value_masks_match": true, + "base_limits": { + "repeatability": { + "atol": 1e-07, + "rtol": 1e-06, + "relative_l2": 1e-06, + "maximum_bad_fraction": 0.0 + }, + "cross_environment": { + "atol": 1e-06, + "rtol": 1e-05, + "relative_l2": 2e-05, + "maximum_bad_fraction": 0.0 + } + }, + "hard_limits": { + "atol": 0.001, + "rtol": 0.01, + "relative_l2": 0.01, + "maximum_bad_fraction": 0.01 + } + }, + "numeric.float16.v1": { + "comparison_type": "numeric", + "description": "Raw FP16 or AMP FP16 outputs", + "locked_fields": [ + "comparison_type", + "base_limits", + "hard_limits", + "require_shape_match", + "require_dtype_match", + "require_exceptional_value_masks_match" + ], + "require_shape_match": true, + "require_dtype_match": true, + "require_exceptional_value_masks_match": true, + "base_limits": { + "repeatability": { + "atol": 0.0005, + "rtol": 0.002, + "relative_l2": 0.002, + "maximum_bad_fraction": 0.0 + }, + "cross_environment": { + "atol": 0.001, + "rtol": 0.005, + "relative_l2": 0.005, + "maximum_bad_fraction": 0.001 + } + }, + "hard_limits": { + "atol": 0.05, + "rtol": 0.1, + "relative_l2": 0.1, + "maximum_bad_fraction": 0.05 + } + }, + "numeric.bfloat16.v1": { + "comparison_type": "numeric", + "description": "Raw BF16 or AMP BF16 outputs", + "locked_fields": [ + "comparison_type", + "base_limits", + "hard_limits", + "require_shape_match", + "require_dtype_match", + "require_exceptional_value_masks_match" + ], + "require_shape_match": true, + "require_dtype_match": true, + "require_exceptional_value_masks_match": true, + "base_limits": { + "repeatability": { + "atol": 0.002, + "rtol": 0.01, + "relative_l2": 0.01, + "maximum_bad_fraction": 0.0 + }, + "cross_environment": { + "atol": 0.005, + "rtol": 0.02, + "relative_l2": 0.02, + "maximum_bad_fraction": 0.001 + } + }, + "hard_limits": { + "atol": 0.1, + "rtol": 0.2, + "relative_l2": 0.2, + "maximum_bad_fraction": 0.05 + } + }, + "numeric.scalar.v1": { + "comparison_type": "numeric", + "description": "Python and JSON floating-point scalars where no tensor dtype is available", + "locked_fields": [ + "comparison_type", + "base_limits", + "hard_limits", + "require_exceptional_value_masks_match" + ], + "require_exceptional_value_masks_match": true, + "base_limits": { + "repeatability": { + "atol": 1e-10, + "rtol": 1e-07, + "relative_l2": 1e-07, + "maximum_bad_fraction": 0.0 + }, + "cross_environment": { + "atol": 1e-08, + "rtol": 1e-05, + "relative_l2": 1e-05, + "maximum_bad_fraction": 0.0 + } + }, + "hard_limits": { + "atol": 0.01, + "rtol": 0.05, + "relative_l2": 0.05, + "maximum_bad_fraction": 0.0 + } + }, + "numeric.fallback.v1": { + "comparison_type": "numeric", + "description": "Fallback for an unrecognised floating-point dtype. This should be reviewed when first encountered", + "locked_fields": [ + "comparison_type", + "base_limits", + "hard_limits", + "require_shape_match", + "require_dtype_match", + "require_exceptional_value_masks_match" + ], + "require_shape_match": true, + "require_dtype_match": true, + "require_exceptional_value_masks_match": true, + "base_limits": { + "repeatability": { + "atol": 1e-06, + "rtol": 1e-05, + "relative_l2": 1e-05, + "maximum_bad_fraction": 0.0 + }, + "cross_environment": { + "atol": 1e-05, + "rtol": 0.0001, + "relative_l2": 0.0001, + "maximum_bad_fraction": 0.001 + } + }, + "hard_limits": { + "atol": 0.1, + "rtol": 0.2, + "relative_l2": 0.2, + "maximum_bad_fraction": 0.05 + } + } + }, + "output_policies": {}, + "calibration": {}, + "reporting": { + "training_progress_checkpoint_count": 5, + "include_initial_checkpoint": true, + "inference_preview_value_count": 512, + "inference_scatter_point_count": 2000, + "inference_sample_error_count": 256, + "training_loss_step_count": 30 + } +} diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/level_0_first_look.py b/pytorch/pytorch_extended_tests/manual_comparison_stuff/level_0_first_look.py new file mode 100644 index 00000000..37190a9b --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/level_0_first_look.py @@ -0,0 +1,693 @@ +#!/usr/bin/env python3 +"""Collate and roughly compare Level 0 summary CSV files. + +USAGE +===== + +1. Create a folder named ``level_0_summaries`` beside the repository root, or + pass another folder as the first positional argument + +2. Put one ``level_0_summary.csv`` result from each CI environment in that + folder and rename each file to describe the environment, for example:: + + level_0_summaries/ + ├── reference.csv + ├── gcc_a100.csv + ├── clang_a100.csv + └── gcc_h100.csv + + The reference file must have the stem ``reference``. The match is + case-insensitive, so ``Reference.csv`` also works + +3. Run:: + + python manual_comparison_stuff/level_0_first_look.py + + Or pass a different input directory:: + + python manual_comparison_stuff/level_0_first_look.py path/to/level_0_summaries + +The script writes two files into the input directory unless ``--output-dir`` +is supplied: + +``level_0_first_look_collated.md`` + One table per example and precision profile. Each input CSV is one column, + using the CSV filename as the column heading + +``level_0_first_look_summary.md`` + A deliberately rough PASS, FAIL or MAYBE comparison against reference.csv + +This is only a first look. It compares the small scalar summaries and prediction +previews, not the complete tensor artefacts. PASS does not prove numerical +compatibility, and MAYBE is meant to prompt inspection of the raw artefacts +rather than being treated as a failure + +The thresholds are intentionally broad and profile-aware. FP16 and BF16 receive +more room than FP32, and later training values receive more room than the initial +forward pass. The future full comparison harness should replace these heuristics +with the versioned per-output policies and repeatability analysis + +The process exits with code 1 when any candidate has an overall FAIL result, +code 2 for invalid input, and code 0 otherwise +""" + +from __future__ import annotations + +import argparse +import csv +import math +import sys +from dataclasses import dataclass +from enum import IntEnum +from pathlib import Path +from typing import Iterable, Mapping + + +DEFAULT_INPUT_DIRECTORY = Path("level_0_summaries") +COLLATED_FILENAME = "level_0_first_look_collated.md" +SUMMARY_FILENAME = "level_0_first_look_summary.md" +IDENTITY_FIELDS = ("test_id", "case_id", "profile_id") +IGNORED_COMPARISON_FIELDS = {"device", "reason"} +EXACT_FIELDS = { + "model_type", + "optimiser", + "training_steps", + "dtype", + "sample_count", + "class_count", + "activation_count", + "activation_names", +} +PREDICTION_FIELDS = {"initial_predictions", "final_predictions"} +ACCURACY_FIELDS = {"initial_accuracy", "final_accuracy"} +LATER_TRAINING_FIELDS = { + "final_loss", + "loss_change", + "final_logits_mean", + "final_logits_standard_deviation", + "final_logits_maximum_absolute", + "final_parameter_l2", +} + + +class Verdict(IntEnum): + """Ordered verdict so the worst result wins cleanly.""" + + PASS = 0 + MAYBE = 1 + FAIL = 2 + + @property + def label(self) -> str: + return self.name + + +@dataclass(frozen=True, slots=True) +class Tolerance: + """Tight and loose scalar limits for one execution profile.""" + + tight_relative: float + tight_absolute: float + loose_relative: float + loose_absolute: float + + +@dataclass(frozen=True, slots=True) +class CsvRun: + """One parsed Level 0 summary CSV.""" + + path: Path + fieldnames: tuple[str, ...] + rows: Mapping[tuple[str, str, str], Mapping[str, str]] + + @property + def name(self) -> str: + return self.path.name + + +@dataclass(frozen=True, slots=True) +class RowComparison: + """Rough comparison for one example/profile row.""" + + verdict: Verdict + reasons: tuple[str, ...] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "input_directory", + nargs="?", + type=Path, + default=DEFAULT_INPUT_DIRECTORY, + ) + parser.add_argument( + "--output-dir", + type=Path, + help="Write the two Markdown files somewhere other than the input directory", + ) + return parser.parse_args() + + +def _normalise_cell(value: str | None) -> str: + return "" if value is None else value.strip() + + +def load_csv(path: Path) -> CsvRun: + try: + with path.open("r", encoding="utf-8", newline="") as source: + reader = csv.DictReader(source) + if reader.fieldnames is None: + raise ValueError("the CSV has no header") + fieldnames = tuple(_normalise_cell(name) for name in reader.fieldnames) + missing_identity = set(IDENTITY_FIELDS) - set(fieldnames) + if missing_identity: + raise ValueError( + f"the CSV is missing identity fields: {sorted(missing_identity)}" + ) + + rows: dict[tuple[str, str, str], Mapping[str, str]] = {} + for line_number, raw_row in enumerate(reader, start=2): + row = {str(key): _normalise_cell(value) for key, value in raw_row.items()} + key = tuple(row[field] for field in IDENTITY_FIELDS) + if any(not part for part in key): + raise ValueError(f"line {line_number} has an empty row identity") + if key in rows: + raise ValueError( + f"line {line_number} duplicates row identity {key!r}" + ) + rows[key] = row + except OSError as exc: + raise ValueError(f"could not read {path}: {exc}") from exc + + if not rows: + raise ValueError(f"{path} contains no result rows") + return CsvRun(path=path, fieldnames=fieldnames, rows=rows) + + +def discover_runs(input_directory: Path) -> tuple[CsvRun, tuple[CsvRun, ...]]: + if not input_directory.is_dir(): + raise ValueError(f"input directory does not exist: {input_directory}") + + paths = sorted(input_directory.glob("*.csv"), key=lambda item: item.name.lower()) + if not paths: + raise ValueError(f"no CSV files were found in {input_directory}") + + reference_paths = [path for path in paths if path.stem.lower() == "reference"] + if len(reference_paths) != 1: + raise ValueError( + "the input directory must contain exactly one CSV with the stem 'reference'" + ) + + reference = load_csv(reference_paths[0]) + candidates = tuple(load_csv(path) for path in paths if path != reference_paths[0]) + return reference, candidates + + +def _markdown(value: object) -> str: + text = "—" if value is None or str(value) == "" else str(value) + return text.replace("|", "\\|").replace("\n", "
") + + +def _heading_for_key(key: tuple[str, str, str]) -> str: + test_id, case_id, profile_id = key + return f"{case_id} — {profile_id}" + + +def _ordered_row_keys(runs: Iterable[CsvRun], reference: CsvRun) -> tuple[tuple[str, str, str], ...]: + ordered = list(reference.rows) + known = set(ordered) + for run in runs: + for key in run.rows: + if key not in known: + ordered.append(key) + known.add(key) + return tuple(ordered) + + +def _ordered_fields(runs: Iterable[CsvRun], reference: CsvRun) -> tuple[str, ...]: + ordered = [field for field in reference.fieldnames if field not in IDENTITY_FIELDS] + known = set(ordered) + for run in runs: + for field in run.fieldnames: + if field not in IDENTITY_FIELDS and field not in known: + ordered.append(field) + known.add(field) + return tuple(ordered) + + +def write_collated( + path: Path, + *, + reference: CsvRun, + candidates: tuple[CsvRun, ...], +) -> None: + runs = (reference, *candidates) + row_keys = _ordered_row_keys(runs, reference) + fields = _ordered_fields(runs, reference) + + lines = [ + "# Level 0 first-look collated results", + "", + "Each input CSV is shown as one column. Missing rows or fields are shown as —", + "", + ] + for key in row_keys: + lines.extend( + [ + f"## {_markdown(_heading_for_key(key))}", + "", + "| Metric | " + " | ".join(_markdown(run.name) for run in runs) + " |", + "|---|" + "---|" * len(runs), + ] + ) + for field in fields: + values = [] + for run in runs: + row = run.rows.get(key) + values.append(None if row is None else row.get(field, "")) + lines.append( + f"| {_markdown(field)} | " + + " | ".join(_markdown(value) for value in values) + + " |" + ) + lines.append("") + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + +def _profile_tolerance(profile_id: str, *, later_training: bool) -> Tolerance: + normalised = profile_id.lower() + if "bfloat16" in normalised: + tolerance = Tolerance(2e-2, 2e-3, 1e-1, 1e-2) + elif "fp16" in normalised: + tolerance = Tolerance(8e-3, 5e-4, 5e-2, 3e-3) + elif "fp64" in normalised: + tolerance = Tolerance(1e-8, 1e-10, 1e-6, 1e-8) + else: + tolerance = Tolerance(2e-4, 2e-6, 3e-3, 3e-5) + + if not later_training: + return tolerance + return Tolerance( + tight_relative=tolerance.tight_relative * 2, + tight_absolute=tolerance.tight_absolute * 2, + loose_relative=tolerance.loose_relative * 2, + loose_absolute=tolerance.loose_absolute * 2, + ) + + +def _parse_float(value: str, *, field: str) -> float: + try: + parsed = float(value) + except ValueError as exc: + raise ValueError(f"{field} is not numeric: {value!r}") from exc + if not math.isfinite(parsed): + raise ValueError(f"{field} is not finite: {value!r}") + return parsed + + +def _within(actual: float, reference: float, *, relative: float, absolute: float) -> bool: + return abs(actual - reference) <= absolute + relative * abs(reference) + + +def _compare_numeric( + field: str, + reference_value: str, + candidate_value: str, + *, + profile_id: str, +) -> RowComparison: + try: + reference_number = _parse_float(reference_value, field=field) + candidate_number = _parse_float(candidate_value, field=field) + except ValueError as exc: + return RowComparison(Verdict.FAIL, (str(exc),)) + + tolerance = _profile_tolerance( + profile_id, + later_training=field in LATER_TRAINING_FIELDS, + ) + if _within( + candidate_number, + reference_number, + relative=tolerance.tight_relative, + absolute=tolerance.tight_absolute, + ): + return RowComparison(Verdict.PASS, ()) + if _within( + candidate_number, + reference_number, + relative=tolerance.loose_relative, + absolute=tolerance.loose_absolute, + ): + return RowComparison( + Verdict.MAYBE, + ( + f"{field} differs from reference by " + f"{candidate_number - reference_number:.6g}", + ), + ) + return RowComparison( + Verdict.FAIL, + ( + f"{field} is outside the rough limit " + f"({reference_number:.6g} vs {candidate_number:.6g})", + ), + ) + + +def _prediction_values(value: str) -> tuple[str, ...]: + return tuple(part for part in value.split() if part) + + +def _compare_prediction_preview(field: str, reference_value: str, candidate_value: str) -> RowComparison: + reference_predictions = _prediction_values(reference_value) + candidate_predictions = _prediction_values(candidate_value) + if len(reference_predictions) != len(candidate_predictions): + return RowComparison( + Verdict.FAIL, + (f"{field} preview length differs from reference",), + ) + differences = sum( + reference_item != candidate_item + for reference_item, candidate_item in zip( + reference_predictions, + candidate_predictions, + ) + ) + if differences == 0: + return RowComparison(Verdict.PASS, ()) + if differences == 1: + return RowComparison( + Verdict.MAYBE, + (f"{field} differs at one preview position",), + ) + return RowComparison( + Verdict.FAIL, + (f"{field} differs at {differences} preview positions",), + ) + + +def _compare_accuracy( + field: str, + reference_value: str, + candidate_value: str, + *, + sample_count: int, +) -> RowComparison: + try: + reference_number = _parse_float(reference_value, field=field) + candidate_number = _parse_float(candidate_value, field=field) + except ValueError as exc: + return RowComparison(Verdict.FAIL, (str(exc),)) + + difference = abs(candidate_number - reference_number) + if difference <= 1e-12: + return RowComparison(Verdict.PASS, ()) + one_sample = 1.0 / max(sample_count, 1) + if difference <= 2 * one_sample + 1e-12: + return RowComparison( + Verdict.MAYBE, + (f"{field} differs by {difference:.6g}",), + ) + return RowComparison( + Verdict.FAIL, + (f"{field} differs by {difference:.6g}",), + ) + + +def _combine(parts: Iterable[RowComparison]) -> RowComparison: + verdict = Verdict.PASS + reasons: list[str] = [] + for part in parts: + verdict = max(verdict, part.verdict) + reasons.extend(part.reasons) + return RowComparison(verdict, tuple(reasons)) + + +def compare_rows( + reference_row: Mapping[str, str], + candidate_row: Mapping[str, str], +) -> RowComparison: + profile_id = reference_row["profile_id"] + parts: list[RowComparison] = [] + + if reference_row.get("status") != "passed": + parts.append( + RowComparison( + Verdict.FAIL, + (f"reference status is {reference_row.get('status') or 'missing'}",), + ) + ) + if candidate_row.get("status") != "passed": + parts.append( + RowComparison( + Verdict.FAIL, + (f"candidate status is {candidate_row.get('status') or 'missing'}",), + ) + ) + + for field in sorted(EXACT_FIELDS): + if field not in reference_row or field not in candidate_row: + parts.append(RowComparison(Verdict.FAIL, (f"{field} is missing",))) + elif reference_row[field] != candidate_row[field]: + parts.append( + RowComparison( + Verdict.FAIL, + (f"{field} differs from reference",), + ) + ) + + for field in sorted(PREDICTION_FIELDS): + if field not in reference_row or field not in candidate_row: + parts.append(RowComparison(Verdict.FAIL, (f"{field} is missing",))) + else: + parts.append( + _compare_prediction_preview( + field, + reference_row[field], + candidate_row[field], + ) + ) + + try: + sample_count = int(reference_row.get("sample_count", "0")) + except ValueError: + sample_count = 0 + for field in sorted(ACCURACY_FIELDS): + if field not in reference_row or field not in candidate_row: + parts.append(RowComparison(Verdict.FAIL, (f"{field} is missing",))) + else: + parts.append( + _compare_accuracy( + field, + reference_row[field], + candidate_row[field], + sample_count=sample_count, + ) + ) + + if "prediction_changes" not in reference_row or "prediction_changes" not in candidate_row: + parts.append(RowComparison(Verdict.FAIL, ("prediction_changes is missing",))) + else: + try: + reference_changes = int(reference_row["prediction_changes"]) + candidate_changes = int(candidate_row["prediction_changes"]) + except ValueError: + parts.append( + RowComparison(Verdict.FAIL, ("prediction_changes is not an integer",)) + ) + else: + difference = abs(candidate_changes - reference_changes) + loose_difference = max(1, math.ceil(max(sample_count, 1) * 0.05)) + if difference == 0: + parts.append(RowComparison(Verdict.PASS, ())) + elif difference <= loose_difference: + parts.append( + RowComparison( + Verdict.MAYBE, + (f"prediction_changes differs by {difference}",), + ) + ) + else: + parts.append( + RowComparison( + Verdict.FAIL, + (f"prediction_changes differs by {difference}",), + ) + ) + + handled = ( + set(IDENTITY_FIELDS) + | IGNORED_COMPARISON_FIELDS + | EXACT_FIELDS + | PREDICTION_FIELDS + | ACCURACY_FIELDS + | {"status", "prediction_changes"} + ) + numeric_fields = sorted( + field + for field in set(reference_row) | set(candidate_row) + if field not in handled + ) + for field in numeric_fields: + reference_value = reference_row.get(field, "") + candidate_value = candidate_row.get(field, "") + if not reference_value or not candidate_value: + parts.append(RowComparison(Verdict.FAIL, (f"{field} is missing",))) + continue + parts.append( + _compare_numeric( + field, + reference_value, + candidate_value, + profile_id=profile_id, + ) + ) + + return _combine(parts) + + +def compare_run(reference: CsvRun, candidate: CsvRun) -> Mapping[tuple[str, str, str], RowComparison]: + output: dict[tuple[str, str, str], RowComparison] = {} + all_keys = tuple(dict.fromkeys((*reference.rows, *candidate.rows))) + for key in all_keys: + reference_row = reference.rows.get(key) + candidate_row = candidate.rows.get(key) + if reference_row is None: + output[key] = RowComparison( + Verdict.MAYBE, + ("candidate has an extra example/profile not present in reference",), + ) + elif candidate_row is None: + output[key] = RowComparison( + Verdict.FAIL, + ("candidate is missing this reference example/profile",), + ) + else: + output[key] = compare_rows(reference_row, candidate_row) + return output + + +def _short_reasons(reasons: tuple[str, ...], *, limit: int = 4) -> str: + if not reasons: + return "—" + selected = list(dict.fromkeys(reasons)) + if len(selected) > limit: + selected = [*selected[:limit], f"and {len(selected) - limit} more"] + return "; ".join(selected) + + +def write_summary( + path: Path, + *, + reference: CsvRun, + candidates: tuple[CsvRun, ...], +) -> bool: + comparisons = { + candidate.name: compare_run(reference, candidate) + for candidate in candidates + } + lines = [ + "# Level 0 first-look comparison", + "", + f"Reference: `{reference.name}`", + "", + "> This is a rough comparison of the summary CSV only. It is not a substitute for comparing the stored tensors or checking repeatability across several runs", + "", + "## Overall", + "", + "| CSV | Verdict | PASS rows | MAYBE rows | FAIL rows |", + "|---|---:|---:|---:|---:|", + f"| {_markdown(reference.name)} | PASS | {len(reference.rows)} | 0 | 0 |", + ] + + any_fail = False + for candidate in candidates: + row_results = comparisons[candidate.name] + counts = {verdict: 0 for verdict in Verdict} + for result in row_results.values(): + counts[result.verdict] += 1 + overall = max( + (result.verdict for result in row_results.values()), + default=Verdict.FAIL, + ) + any_fail = any_fail or overall == Verdict.FAIL + lines.append( + f"| {_markdown(candidate.name)} | {overall.label} | " + f"{counts[Verdict.PASS]} | {counts[Verdict.MAYBE]} | " + f"{counts[Verdict.FAIL]} |" + ) + + if not candidates: + lines.extend( + [ + "", + "No candidate CSV files were found, so only the reference was collated", + ] + ) + else: + lines.extend( + [ + "", + "## Per-example results", + "", + "| CSV | Example | Profile | Verdict | Main reasons |", + "|---|---|---|---:|---|", + ] + ) + for candidate in candidates: + for key, result in comparisons[candidate.name].items(): + _, case_id, profile_id = key + lines.append( + f"| {_markdown(candidate.name)} | {_markdown(case_id)} | " + f"{_markdown(profile_id)} | {result.verdict.label} | " + f"{_markdown(_short_reasons(result.reasons))} |" + ) + + lines.extend( + [ + "", + "## Interpretation", + "", + "- **PASS** means the small summary values are close to the reference under deliberately rough profile-aware thresholds", + "- **MAYBE** means the result is plausible but different enough that the detailed tensor artefacts should be inspected", + "- **FAIL** means a row is missing, structurally different, did not pass in CI, or is well outside the rough summary thresholds", + ] + ) + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + return any_fail + + +def main() -> int: + args = parse_args() + input_directory = args.input_directory.resolve() + output_directory = ( + args.output_dir.resolve() if args.output_dir is not None else input_directory + ) + + try: + reference, candidates = discover_runs(input_directory) + collated_path = output_directory / COLLATED_FILENAME + summary_path = output_directory / SUMMARY_FILENAME + write_collated(collated_path, reference=reference, candidates=candidates) + any_fail = write_summary( + summary_path, + reference=reference, + candidates=candidates, + ) + except ValueError as exc: + print(f"level_0_first_look: {exc}", file=sys.stderr) + return 2 + + print(f"Wrote {collated_path}") + print(f"Wrote {summary_path}") + return 1 if any_fail else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/manual_comparison_stuff/manual-how-to.md b/pytorch/pytorch_extended_tests/manual_comparison_stuff/manual-how-to.md new file mode 100644 index 00000000..1b80ed47 --- /dev/null +++ b/pytorch/pytorch_extended_tests/manual_comparison_stuff/manual-how-to.md @@ -0,0 +1,408 @@ +# Running the comparison tools on Linux + +This starts from an empty working folder.It only needs Python, numpy, and matplotlib. + +## 1. Make working folder + +```bash +mkdir -p ~/pytorch-comparisons +cd ~/pytorch-comparisons +``` + +## 2. Copy the required files from `pytorch_extended_tests` + +Copy these two folders without flattening them: + +```text +pytorch-comparisons/ +├── config/ +│ ├── __init__.py +│ ├── suite_config.py +│ └── test_catalogue.py +└── manual_comparison_stuff/ + ├── analyse_repeatability.py + ├── compare_environment_outputs.py + ├── compare_repeatability_analyses.py + ├── comparison_policy.py + ├── comparison_policy_template.json + └── level_0_first_look.py +``` + +Example: + +```bash +cp -a /path/to/pytorch_extended_tests/config . +cp -a /path/to/pytorch_extended_tests/manual_comparison_stuff . +``` + +Keep the folders beside each other. `analyse_repeatability.py` uses `config/test_catalogue.py` to recover the level, category and output metadata. + +## 3. Create and activate the virtual environment + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install numpy matplotlib +``` + +Check the imports if you want to be sure: + +```bash +python -c "import numpy, matplotlib; print('Comparison environment ready')" +``` + +Reactivate the environment in a later shell with: + +```bash +cd ~/pytorch-comparisons +source .venv/bin/activate +``` + +--- + +# A. Summary comparison workflow + +This compares repeatability-analysis json files rather than loading every raw tensor for the cross-environment comparison. + +It is quicker and smaller, but a changed tensor hash may be reported as `MAYBE` because the JSON does not contain the tensor values. +Might be better for nightlies? + +## A1. Arrange the raw CI outputs + +Create one folder per environment: + +```bash +mkdir -p summary_workflow/raw/reference +mkdir -p summary_workflow/raw/a100_gcc +mkdir -p summary_workflow/raw/h100_clang +``` + +Copy each complete CI output into a run folder whose name starts with `repeatability_`: + +```text +summary_workflow/ +└── raw/ + ├── reference/ + │ ├── repeatability_001/ + │ ├── repeatability_002/ + │ └── repeatability_003/ + ├── a100_gcc/ + │ ├── repeatability_001/ + │ ├── repeatability_002/ + │ └── repeatability_003/ + └── h100_clang/ + ├── repeatability_001/ + ├── repeatability_002/ + └── repeatability_003/ +``` + +Each `repeatability_*` folder must directly contain the unmodified suite result bundle: + +```text +run_manifest.json +observations.jsonl +test_status.json +artifacts/ +``` + +`execution.log`, `level_0_summary.csv` and the other files may remain in the bundle. + +Use environment names which identify the GPU and build, for example: + +```text +a100_gcc +a100_clang +h100_gcc +cpu_reference +``` + +Do not combine runs from different environments in the same environment folder. + +## A2. Analyse repeatability for the reference + +```bash +python manual_comparison_stuff/analyse_repeatability.py \ + summary_workflow/raw/reference \ + --write-populated-policy +``` + +This writes: + +```text +summary_workflow/raw/reference/repeatability_analysis/ +├── repeatability_analysis.json +├── repeatability_analysis.md +├── comparison_policy.json +└── *.png +``` + +## A3. Analyse repeatability for every candidate + +```bash +python manual_comparison_stuff/analyse_repeatability.py \ + summary_workflow/raw/a100_gcc + +python manual_comparison_stuff/analyse_repeatability.py \ + summary_workflow/raw/h100_clang +``` + +Repeat this command for each additional environment. + +## A4. Collect the analysis JSON files + +```bash +mkdir -p summary_workflow/repeatability_outputs + +cp \ + summary_workflow/raw/reference/repeatability_analysis/repeatability_analysis.json \ + summary_workflow/repeatability_outputs/reference.json + +cp \ + summary_workflow/raw/a100_gcc/repeatability_analysis/repeatability_analysis.json \ + summary_workflow/repeatability_outputs/a100_gcc.json + +cp \ + summary_workflow/raw/h100_clang/repeatability_analysis/repeatability_analysis.json \ + summary_workflow/repeatability_outputs/h100_clang.json + +cp \ + summary_workflow/raw/reference/repeatability_analysis/comparison_policy.json \ + summary_workflow/repeatability_outputs/comparison_policy.json +``` + +The reference analysis file must be named exactly: + +```text +reference.json +``` + +The other JSON filenames become the candidate names in the report. + +The resulting structure should be: + +```text +summary_workflow/ +└── repeatability_outputs/ + ├── reference.json + ├── scale-gfx1100.json + ├── scale-gfx1201.json + └── comparison_policy.json +``` + +## A5. Compare the repeatability analyses + +```bash +python manual_comparison_stuff/compare_repeatability_analyses.py \ + summary_workflow/repeatability_outputs \ + --policy summary_workflow/repeatability_outputs/comparison_policy.json +``` + +Outputs: + +```text +summary_workflow/repeatability_outputs/repeatability_comparison/ +├── repeatability_comparison_results.json +├── repeatability_comparison_results.md +└── repeatability_comparison_status.png +``` + +Open the Markdown report with, for example: + +```bash +less summary_workflow/repeatability_outputs/repeatability_comparison/repeatability_comparison_results.md +``` + +## A6. Optional Level 0 CSV first look + +Use one `level_0_summary.csv` per environment: + +```bash +mkdir -p level_0_summaries + +cp /path/to/reference_run/level_0_summary.csv level_0_summaries/reference.csv +cp /path/to/a100_gcc_run/level_0_summary.csv level_0_summaries/a100_gcc.csv +cp /path/to/h100_clang_run/level_0_summary.csv level_0_summaries/h100_clang.csv +``` + +Run: + +```bash +python manual_comparison_stuff/level_0_first_look.py level_0_summaries +``` + +Outputs: + +```text +level_0_summaries/ +├── level_0_first_look_collated.md +└── level_0_first_look_summary.md +``` + +This is deliberately rough. Use the detailed workflow for the real tensor comparison. + +--- + +# B. Detailed comparison workflow + +This compares the raw scalar values and tensor artefacts from every candidate run against every reference run. + +It also analyses repeatability for every environment and populates the comparison policy from the reference runs. + +## B1. Create the comparison root + +```bash +mkdir -p detailed_comparison/reference +mkdir -p detailed_comparison/a100_gcc +mkdir -p detailed_comparison/h100_clang +``` + +Copy the policy template: + +```bash +cp \ + manual_comparison_stuff/comparison_policy_template.json \ + detailed_comparison/comparison_policy_template.json +``` + +## B2. Copy the raw CI outputs + +The reference environment folder must be named: + +```text +reference +``` + +Candidate environment folders can use any clear name. + +Run-folder names are unimportant in this workflow. Use a consistent convention such as `run_001`. + +```text +detailed_comparison/ +├── comparison_policy_template.json +├── reference/ +│ ├── run_001/ +│ ├── run_002/ +│ └── run_003/ +├── scale-gfx1100/ +│ ├── run_001/ +│ ├── run_002/ +│ └── run_003/ +└── scale-gfx1201/ + ├── run_001/ + ├── run_002/ + └── run_003/ +``` + +Each run folder must directly contain: + +```text +run_manifest.json +observations.jsonl +test_status.json +artifacts/ +``` + +Do not copy only `level_0_summary.csv`. The detailed comparison needs the complete result bundle. + +## B3. Run the detailed comparison + +```bash +python manual_comparison_stuff/compare_environment_outputs.py \ + detailed_comparison +``` + +The script will: + +- analyse repeatability within `reference` +- analyse repeatability within each candidate environment +- populate the policy from the reference repeatability +- compare every candidate run with every reference run +- verify tensor artefact hashes +- apply the populated exact and numerical policies +- write JSON, Markdown and Matplotlib reports + +Outputs: + +```text +detailed_comparison/ +├── comparison_policy.json +└── comparison_results/ + ├── comparison_results.json + ├── comparison_results.md + ├── environment_status_counts.png + └── environment_worst_tolerance_ratio.png +``` + +Open the report: + +```bash +less detailed_comparison/comparison_results/comparison_results.md +``` + +## B4. Useful optional flags + +Keep every individual reference/candidate pair in the JSON: + +```bash +python manual_comparison_stuff/compare_environment_outputs.py \ + detailed_comparison \ + --retain-all-pairs +``` + +Skip plots: + +```bash +python manual_comparison_stuff/compare_environment_outputs.py \ + detailed_comparison \ + --no-plots +``` + +Use a different reference-folder name: + +```bash +python manual_comparison_stuff/compare_environment_outputs.py \ + detailed_comparison \ + --reference-folder known_good_gpu +``` + +Normally, keeping the folder name `reference` is simpler. + +--- + +# Multiple precision profiles + +Multiple precision profiles can be included in the same runs. + +For example, every CUDA environment may contain: + +```text +controlled_fp32 +amp_fp16 +``` + +The tools handle this as follows: + +- `profile_id` is part of every output identity +- FP32 outputs are compared only with FP32 outputs +- AMP FP16 outputs are compared only with AMP FP16 outputs +- repeatability is measured separately for each profile +- the generated policy contains separate calibrated output entries for each profile +- the Markdown and JSON reports retain the profile name + +Requirements: + +- every repeat within one environment must use the same profile list +- the reference and all candidate environments must use the same profile list +- use the same profile order as well +- use the same levels, cases, seed, catalogue version and prepared datasets +- `controlled_fp16` and `amp_fp16` are different profiles and are not interchangeable + +A profile-set mismatch is treated as incompatible metadata and makes the environment comparison fail. + +E.g.: + +- CPU versus GPU: run both with `controlled_fp32` only +- known-good combo versus other compiler-GPU-combos: run all environments with `controlled_fp32 amp_fp16` +- BF16 comparison: opt every compared GPU into the same BF16 profile +- if two environments do not support the same profiles, run separate comparison roots containing only their shared profile set \ No newline at end of file diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/__init__.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/__init__.py new file mode 100644 index 00000000..d9936bd0 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/__init__.py @@ -0,0 +1,5 @@ +"""Extended numerical and workload tests for PyTorch builds.""" + +from config.suite_config import RESULT_FORMAT_VERSION, SUITE_NAME, SUITE_VERSION + +__all__ = ["RESULT_FORMAT_VERSION", "SUITE_NAME", "SUITE_VERSION"] diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/case_api.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/case_api.py new file mode 100644 index 00000000..3cabe581 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/case_api.py @@ -0,0 +1,87 @@ +"""Small public API used by the individual test case modules.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Protocol + +from config.suite_config import DATASET_PATHS, derive_seed + + +class UnsupportedCase(RuntimeError): + """Raised by a case when the current backend cannot support it.""" + + +class ObservationRecorder(Protocol): + """Interface exposed to case modules for recording named outputs.""" + + def record( + self, + output_id: str, + value: Any, + *, + coordinates: Mapping[str, Any] | None = None, + ) -> None: + """Record one output declared in the test catalogue.""" + + +@dataclass(frozen=True, slots=True) +class CaseContext: + """Execution details supplied to one case function.""" + + test_id: str + case_id: str + profile_id: str + device: str + dtype_name: str + autocast_dtype_name: str | None + seed: int + temporary_directory: Path + + def dataset_path(self, dataset_id: str) -> Path: + """Return the configured prepared directory for a dataset ID.""" + + try: + return DATASET_PATHS[dataset_id] + except KeyError as exc: + raise KeyError(f"Unknown dataset ID: {dataset_id}") from exc + + def seed_for(self, name: str) -> int: + """Derive a stable child seed for a distinct random stream.""" + + if not name: + raise ValueError("Seed stream names must not be empty") + return derive_seed(self.test_id, self.profile_id, self.case_id, name) + + def torch_dtype(self) -> Any: + """Resolve the configured dtype without importing PyTorch in config code.""" + + import torch + + try: + return getattr(torch, self.dtype_name) + except AttributeError as exc: + raise ValueError(f"Unknown PyTorch dtype: {self.dtype_name}") from exc + + def autocast(self) -> AbstractContextManager[Any]: + """Return the configured autocast context, or a no-op context.""" + + if self.autocast_dtype_name is None: + return nullcontext() + + import torch + + try: + autocast_dtype = getattr(torch, self.autocast_dtype_name) + except AttributeError as exc: + raise ValueError( + f"Unknown PyTorch autocast dtype: {self.autocast_dtype_name}" + ) from exc + + return torch.autocast( + device_type=torch.device(self.device).type, + dtype=autocast_dtype, + enabled=True, + ) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/datasets/__init__.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/datasets/__init__.py new file mode 100644 index 00000000..c8133847 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/datasets/__init__.py @@ -0,0 +1,5 @@ +"""Prepared dataset validation used before the suite starts.""" + +from .validation import DatasetValidationError, validate_datasets + +__all__ = ["DatasetValidationError", "validate_datasets"] diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/datasets/validation.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/datasets/validation.py new file mode 100644 index 00000000..7c85beae --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/datasets/validation.py @@ -0,0 +1,166 @@ +"""Validate committed source and prepared datasets against the manifest.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable, Mapping + +from config.suite_config import ( + DATASET_MANIFEST_PATH, + DATASET_PATHS, + DATASETS_DIR, + ROOT_SEED, + SUITE_NAME, + SUITE_VERSION, +) + + +class DatasetValidationError(RuntimeError): + """Raised when committed dataset content does not match its manifest.""" + + +def hash_file(path: Path, algorithm: str = "sha256") -> str: + digest = hashlib.new(algorithm) + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_dataset_manifest(path: Path = DATASET_MANIFEST_PATH) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise DatasetValidationError(f"Dataset manifest does not exist: {path}") from exc + except json.JSONDecodeError as exc: + raise DatasetValidationError(f"Dataset manifest is not valid JSON: {path}") from exc + + if not isinstance(value, dict): + raise DatasetValidationError("Dataset manifest must contain a JSON object") + return value + + +def _validate_file_record(record: Mapping[str, Any], *, label: str) -> None: + relative_path = record.get("relative_path") + if not isinstance(relative_path, str) or not relative_path: + raise DatasetValidationError(f"{label} has no valid relative_path") + + path = DATASETS_DIR / relative_path + if not path.is_file(): + raise DatasetValidationError(f"Required dataset file is missing: {path}") + + expected_size = record.get("size_bytes") + if not isinstance(expected_size, int): + raise DatasetValidationError( + f"{label} has no recorded size\n" + "Run datasets/generate_datasets.py after adding the downloaded files" + ) + actual_size = path.stat().st_size + if actual_size != expected_size: + raise DatasetValidationError( + f"Dataset file size does not match the manifest: {path}\n" + f"Expected {expected_size}, found {actual_size}" + ) + + expected_sha256 = record.get("sha256") + if not isinstance(expected_sha256, str) or len(expected_sha256) != 64: + raise DatasetValidationError( + f"{label} has no recorded SHA-256 hash\n" + "Run datasets/generate_datasets.py after adding the downloaded files" + ) + actual_sha256 = hash_file(path) + if actual_sha256 != expected_sha256: + raise DatasetValidationError( + f"Dataset file hash does not match the manifest: {path}\n" + f"Expected {expected_sha256}, found {actual_sha256}" + ) + + expected_md5 = record.get("expected_md5") + if isinstance(expected_md5, str): + actual_md5 = hash_file(path, "md5") + if actual_md5 != expected_md5: + raise DatasetValidationError( + f"Dataset file does not match the publisher MD5: {path}\n" + f"Expected {expected_md5}, found {actual_md5}" + ) + + +def _dataset_entry(manifest: Mapping[str, Any], dataset_id: str) -> Mapping[str, Any]: + if dataset_id in manifest.get("generated_datasets", {}): + return manifest["generated_datasets"][dataset_id] + if dataset_id in manifest.get("prepared_datasets", {}): + return manifest["prepared_datasets"][dataset_id] + raise DatasetValidationError(f"Dataset manifest has no entry for {dataset_id}") + + +def validate_datasets( + required_dataset_ids: Iterable[str], + *, + validate_downloaded_sources: bool, +) -> str: + """Validate required prepared data and return the manifest SHA-256.""" + + manifest = load_dataset_manifest() + if manifest.get("suite_name") != SUITE_NAME: + raise DatasetValidationError("Dataset manifest suite_name does not match this suite") + if manifest.get("suite_version") != SUITE_VERSION: + raise DatasetValidationError("Dataset manifest suite_version does not match this suite") + if manifest.get("root_seed") != ROOT_SEED: + raise DatasetValidationError( + "Dataset manifest root_seed does not match config/suite_config.py" + ) + + requested = tuple(dict.fromkeys(required_dataset_ids)) + unknown = set(requested) - set(DATASET_PATHS) + if unknown: + raise DatasetValidationError(f"Unknown required dataset IDs: {sorted(unknown)}") + + for dataset_id in requested: + configured_path = DATASET_PATHS[dataset_id] + if not configured_path.is_dir(): + raise DatasetValidationError( + f"Prepared dataset directory is missing: {configured_path}\n" + "Run datasets/generate_datasets.py and commit the generated files" + ) + + entry = _dataset_entry(manifest, dataset_id) + files = entry.get("files") + if not isinstance(files, list) or not files: + raise DatasetValidationError( + f"Dataset manifest has no prepared files for {dataset_id}\n" + "Run datasets/generate_datasets.py and commit the updated manifest" + ) + for index, record in enumerate(files): + if not isinstance(record, Mapping): + raise DatasetValidationError(f"Invalid file record for {dataset_id}") + _validate_file_record(record, label=f"{dataset_id} file {index}") + + if validate_downloaded_sources: + sources = manifest.get("sources") + prepared_entries = manifest.get("prepared_datasets") + if not isinstance(sources, Mapping): + raise DatasetValidationError("Dataset manifest has no sources mapping") + if not isinstance(prepared_entries, Mapping): + raise DatasetValidationError("Dataset manifest has no prepared_datasets mapping") + + required_source_ids = { + prepared_entries[dataset_id]["source_id"] + for dataset_id in requested + if dataset_id in prepared_entries + } + for source_id in required_source_ids: + source = sources.get(source_id) + if not isinstance(source, Mapping): + raise DatasetValidationError(f"Invalid source entry: {source_id}") + files = source.get("files") + if not isinstance(files, list): + raise DatasetValidationError(f"Source has no files list: {source_id}") + for index, record in enumerate(files): + if not isinstance(record, Mapping): + raise DatasetValidationError(f"Invalid source file entry: {source_id}") + if record.get("required", True): + _validate_file_record(record, label=f"{source_id} source file {index}") + + return hash_file(DATASET_MANIFEST_PATH) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/__init__.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/__init__.py new file mode 100644 index 00000000..2d146858 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/__init__.py @@ -0,0 +1 @@ +"""Execution orchestration for the extended test suite.""" diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/execution_plan.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/execution_plan.py new file mode 100644 index 00000000..b860389c --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/execution_plan.py @@ -0,0 +1,120 @@ +"""Build the ordered list of test-module/profile tasks for one invocation.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Iterable + +from config.suite_config import ( + DEFAULT_PROFILES_BY_DEVICE, + EXECUTION, + EXECUTION_PROFILES, + LEVELS, +) +from config.test_catalogue import TEST_CATALOGUE, TestSpec + + +@dataclass(frozen=True, slots=True) +class ExecutionTask: + """One test module running all of its cases under one profile.""" + + task_id: str + test_id: str + profile_id: str + device: str + case_ids: tuple[str, ...] + observations_relative_path: str + status_relative_path: str + + def as_dict(self) -> dict[str, object]: + return asdict(self) + + +def _normalise_selection( + requested: Iterable[str] | None, + *, + defaults: tuple[str, ...], + allowed: set[str], + label: str, +) -> tuple[str, ...]: + values = defaults if requested is None else tuple(dict.fromkeys(requested)) + unknown = set(values) - allowed + if unknown: + raise ValueError(f"Unknown {label}: {sorted(unknown)}") + return tuple(values) + + +def select_test_specs( + *, + levels: Iterable[str] | None, + test_ids: Iterable[str] | None, +) -> tuple[TestSpec, ...]: + selected_levels = _normalise_selection( + levels, + defaults=tuple(EXECUTION["enabled_levels"]), + allowed=set(LEVELS), + label="test levels", + ) + requested_test_ids = None if test_ids is None else tuple(dict.fromkeys(test_ids)) + + configured_enabled = set(EXECUTION["enabled_test_ids"]) + configured_disabled = set(EXECUTION["disabled_test_ids"]) + known_test_ids = {spec.test_id for spec in TEST_CATALOGUE} + + if configured_enabled - known_test_ids: + raise ValueError("EXECUTION enabled_test_ids contains unknown tests") + if configured_disabled - known_test_ids: + raise ValueError("EXECUTION disabled_test_ids contains unknown tests") + if requested_test_ids is not None and set(requested_test_ids) - known_test_ids: + unknown = sorted(set(requested_test_ids) - known_test_ids) + raise ValueError(f"Unknown test IDs: {unknown}") + + selected: list[TestSpec] = [] + for spec in TEST_CATALOGUE: + if spec.level not in selected_levels: + continue + if configured_enabled and spec.test_id not in configured_enabled: + continue + if spec.test_id in configured_disabled: + continue + if requested_test_ids is not None and spec.test_id not in requested_test_ids: + continue + selected.append(spec) + return tuple(selected) + + +def build_execution_plan( + *, + device: str, + profiles: Iterable[str] | None, + levels: Iterable[str] | None, + test_ids: Iterable[str] | None, +) -> tuple[ExecutionTask, ...]: + selected_profiles = _normalise_selection( + profiles, + defaults=tuple(DEFAULT_PROFILES_BY_DEVICE[device]), + allowed=set(EXECUTION_PROFILES), + label="execution profiles", + ) + test_specs = select_test_specs(levels=levels, test_ids=test_ids) + + tasks: list[ExecutionTask] = [] + for spec in test_specs: + for profile_id in selected_profiles: + if profile_id not in spec.profile_ids: + continue + index = len(tasks) + task_name = f"{index:04d}_{spec.test_id}_{profile_id}".replace(".", "_") + task_directory = f".work/tasks/{task_name}" + tasks.append( + ExecutionTask( + task_id=task_name, + test_id=spec.test_id, + profile_id=profile_id, + device=device, + case_ids=spec.case_ids, + observations_relative_path=f"{task_directory}/observations.jsonl", + status_relative_path=f"{task_directory}/task_status.json", + ) + ) + return tuple(tasks) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/run_suite.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/run_suite.py new file mode 100644 index 00000000..93787d91 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/run_suite.py @@ -0,0 +1,162 @@ +"""Run the configured PyTorch cases and write a raw CI result bundle.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Iterable + +from config.suite_config import ( + ALLOWED_DEVICES, + DATASET_MANIFEST_PATH, + DEFAULT_DEVICE, + DEVICE_ENVIRONMENT_VARIABLE, + EXECUTION, + EXECUTION_PROFILES, + RESULTS_DIR, +) +from config.test_catalogue import catalogue_as_dict, get_test_spec +from pytorch_extended_tests.datasets.validation import validate_datasets +from pytorch_extended_tests.orchestrator.execution_plan import build_execution_plan +from pytorch_extended_tests.orchestrator.subprocess_runner import run_task_subprocess +from pytorch_extended_tests.results.result_bundle import ResultBundle + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-dir", type=Path, default=RESULTS_DIR) + parser.add_argument("--device", choices=ALLOWED_DEVICES) + parser.add_argument("--profiles", nargs="+", choices=tuple(EXECUTION_PROFILES)) + parser.add_argument("--levels", nargs="+") + parser.add_argument("--tests", nargs="+") + parser.add_argument( + "--keep-existing", + action="store_true", + help="Do not remove the result directory before starting", + ) + parser.add_argument( + "--list-plan", + action="store_true", + help="Print the selected tasks without running them", + ) + return parser.parse_args() + + +def resolve_device(command_line_device: str | None) -> str: + value = command_line_device or os.environ.get(DEVICE_ENVIRONMENT_VARIABLE) or DEFAULT_DEVICE + if value not in ALLOWED_DEVICES: + raise ValueError( + f"{DEVICE_ENVIRONMENT_VARIABLE} must be one of {', '.join(ALLOWED_DEVICES)}" + ) + return value + + +def required_dataset_ids(test_ids: Iterable[str]) -> tuple[str, ...]: + ordered: list[str] = [] + for test_id in test_ids: + for dataset_id in get_test_spec(test_id).dataset_ids: + if dataset_id not in ordered: + ordered.append(dataset_id) + return tuple(ordered) + + +def print_plan(tasks: Iterable[object]) -> None: + for task in tasks: + print(f"{task.task_id}: {task.test_id} [{task.profile_id}] on {task.device}") + + +def main() -> int: + args = parse_args() + device = resolve_device(args.device) + plan = build_execution_plan( + device=device, + profiles=args.profiles, + levels=args.levels, + test_ids=args.tests, + ) + if not plan: + raise RuntimeError("The selected configuration produced an empty execution plan") + + if args.list_plan: + print_plan(plan) + return 0 + + datasets = required_dataset_ids(task.test_id for task in plan) + try: + validate_datasets( + datasets, + validate_downloaded_sources=bool(EXECUTION["validate_downloaded_sources"]), + ) + except Exception as exc: + args.results_dir.mkdir(parents=True, exist_ok=True) + (args.results_dir / "preflight_error.json").write_text( + json.dumps( + { + "status": "failed", + "stage": "dataset_validation", + "error_type": type(exc).__name__, + "message": str(exc), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + print(f"Dataset validation failed: {exc}", file=sys.stderr) + return 2 + + bundle = ResultBundle( + args.results_dir, + remove_existing=bool(EXECUTION["remove_existing_results"]) and not args.keep_existing, + ) + selected_profiles = tuple(dict.fromkeys(task.profile_id for task in plan)) + bundle.write_initial_manifest( + dataset_manifest_path=DATASET_MANIFEST_PATH, + device=device, + profile_ids=selected_profiles, + planned_task_count=len(plan), + ) + if EXECUTION["write_catalogue_snapshot"]: + (args.results_dir / "test_catalogue.json").write_text( + json.dumps(catalogue_as_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + print(f"Running {len(plan)} test/profile tasks on {device}") + print(f"Writing raw results to {args.results_dir}") + + task_records = [] + try: + for index, task in enumerate(plan, start=1): + print(f"\n[{index}/{len(plan)}] {task.test_id} [{task.profile_id}]") + record = run_task_subprocess( + task, + results_root=args.results_dir, + timeout_seconds=int(EXECUTION["subprocess_timeout_seconds"]), + ) + task_records.append(record) + if record.get("status") in {"failed", "timed_out"}: + print(f"Task failed: {task.test_id} [{task.profile_id}]", file=sys.stderr) + if not EXECUTION["continue_after_test_file_failure"]: + break + except KeyboardInterrupt: + print("Suite interrupted", file=sys.stderr) + overall_status = "failed" + bundle.finalise(task_records=task_records, overall_status=overall_status) + return 130 + + failed = any(record.get("status") in {"failed", "timed_out"} for record in task_records) + incomplete = len(task_records) != len(plan) + overall_status = "failed" if failed or incomplete else "passed" + bundle.finalise(task_records=task_records, overall_status=overall_status) + + print(f"\nSuite execution status: {overall_status}") + return 1 if overall_status == "failed" else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/run_test_file.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/run_test_file.py new file mode 100644 index 00000000..d7546841 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/run_test_file.py @@ -0,0 +1,352 @@ +"""Child-process entry point for one test module and execution profile.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import random +import shutil +import sys +import tempfile +import traceback +from pathlib import Path +from typing import Any, Mapping + +import numpy as np + +from config.suite_config import ( + EXECUTION, + EXECUTION_PROFILES, + derive_seed, +) +from config.test_catalogue import get_test_spec +from pytorch_extended_tests.case_api import CaseContext, UnsupportedCase +from pytorch_extended_tests.results.artifact_writer import CaseObservationWriter +from pytorch_extended_tests.results.observation import CaseExecutionRecord, utc_now +from pytorch_extended_tests.precision_settings import apply_float32_precision + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--task-file", type=Path, required=True) + parser.add_argument("--results-dir", type=Path, required=True) + return parser.parse_args() + + +def load_task(path: Path) -> dict[str, Any]: + try: + task = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError(f"Could not read task file: {path}") from exc + if not isinstance(task, dict): + raise RuntimeError("Task file must contain a JSON object") + return task + + +def configure_torch(profile_id: str, device_name: str) -> tuple[Any, Mapping[str, Any]]: + import torch + + profile = EXECUTION_PROFILES[profile_id] + device = torch.device(device_name) + if device.type == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was selected but torch.cuda.is_available() is false") + + torch.set_num_threads(int(profile["cpu_threads"])) + torch.set_num_interop_threads(int(profile["interop_threads"])) + torch.use_deterministic_algorithms( + bool(profile["deterministic_algorithms"]), + warn_only=bool(profile["deterministic_warn_only"]), + ) + apply_float32_precision( + allow_tf32=bool(profile["allow_tf32"]), + matmul_precision=str(profile["float32_matmul_precision"]), + ) + + if hasattr(torch.backends, "cudnn"): + torch.backends.cudnn.benchmark = bool(profile["cudnn_benchmark"]) + torch.backends.cudnn.deterministic = bool(profile["cudnn_deterministic"]) + + return device, profile + + +def unsupported_profile_reason(profile_id: str, device: Any) -> str | None: + import torch + + profile = EXECUTION_PROFILES[profile_id] + if profile_id == "controlled_fp16" and device.type != "cuda": + return "The raw FP16 profile is only exercised on CUDA in this suite" + if profile_id == "amp_fp16" and device.type != "cuda": + return "FP16 autocast is only exercised on CUDA in this suite" + + autocast_dtype = profile.get("autocast_dtype") + if autocast_dtype is not None: + autocast_available = torch.amp.autocast_mode.is_autocast_available(device.type) + if not autocast_available: + return f"Autocast is not available for the selected {device.type} backend" + + if profile_id in {"controlled_bfloat16", "amp_bfloat16"} and device.type == "cuda": + if not torch.cuda.is_bf16_supported(): + return "The selected CUDA device does not support bfloat16" + return None + + +def seed_everything(seed: int) -> None: + import torch + + random.seed(seed) + np.random.seed(seed % (2**32)) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +def _case_record( + *, + test_id: str, + case_id: str, + profile_id: str, + status: str, + seed: int, + started_at: str, + writer: CaseObservationWriter, + reason: str | None = None, + traceback_text: str | None = None, +) -> CaseExecutionRecord: + return CaseExecutionRecord( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + status=status, + seed=seed, + started_at_utc=started_at, + ended_at_utc=utc_now(), + produced_output_ids=writer.produced_output_ids, + missing_required_output_ids=writer.missing_required_output_ids(), + reason=reason, + traceback=traceback_text, + ) + + +def run_cases(task: Mapping[str, Any], results_root: Path) -> tuple[str, list[dict[str, Any]], str | None]: + test_id = str(task["test_id"]) + profile_id = str(task["profile_id"]) + device_name = str(task["device"]) + test_spec = get_test_spec(test_id) + observations_path = results_root / str(task["observations_relative_path"]) + observations_path.parent.mkdir(parents=True, exist_ok=True) + observations_path.unlink(missing_ok=True) + + device, profile = configure_torch(profile_id, device_name) + profile_reason = unsupported_profile_reason(profile_id, device) + + if profile_reason is not None: + records: list[dict[str, Any]] = [] + for case_id_value in task["case_ids"]: + case_id = str(case_id_value) + started_at = utc_now() + seed = derive_seed(test_id, profile_id, case_id, "case") + writer = CaseObservationWriter( + results_root=results_root, + observations_path=observations_path, + test_spec=test_spec, + case_id=case_id, + profile_id=profile_id, + seed=seed, + ) + writer.record_skipped(profile_reason) + records.append( + _case_record( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + status="skipped_unsupported", + seed=seed, + started_at=started_at, + writer=writer, + reason=profile_reason, + ).as_dict() + ) + return "skipped_unsupported", records, profile_reason + + seed_everything(derive_seed(test_id, profile_id, "module_import")) + try: + module = importlib.import_module(test_spec.module) + run_case = getattr(module, "run_case") + if not callable(run_case): + raise TypeError(f"{test_spec.module}.run_case is not callable") + except Exception: + error_text = traceback.format_exc() + print(error_text, file=sys.stderr) + records = [] + for case_id in task["case_ids"]: + seed = derive_seed(test_id, profile_id, str(case_id), "case") + writer = CaseObservationWriter( + results_root=results_root, + observations_path=observations_path, + test_spec=test_spec, + case_id=str(case_id), + profile_id=profile_id, + seed=seed, + ) + writer.record_failure("Test module could not be imported") + records.append( + _case_record( + test_id=test_id, + case_id=str(case_id), + profile_id=profile_id, + status="failed", + seed=seed, + started_at=utc_now(), + writer=writer, + reason="Test module could not be imported", + traceback_text=error_text, + ).as_dict() + ) + return "failed", records, "Test module could not be imported" + + records = [] + for case_id_value in task["case_ids"]: + case_id = str(case_id_value) + started_at = utc_now() + seed = derive_seed(test_id, profile_id, case_id, "case") + writer = CaseObservationWriter( + results_root=results_root, + observations_path=observations_path, + test_spec=test_spec, + case_id=case_id, + profile_id=profile_id, + seed=seed, + ) + + seed_everything(seed) + temporary_parent = observations_path.parent / "temporary" + temporary_parent.mkdir(parents=True, exist_ok=True) + temporary_directory = Path( + tempfile.mkdtemp(prefix=f"{case_id.replace('.', '_')}-", dir=temporary_parent) + ) + context = CaseContext( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + device=str(device), + dtype_name=str(profile["dtype"]), + autocast_dtype_name=profile["autocast_dtype"], + seed=seed, + temporary_directory=temporary_directory, + ) + + try: + run_case(context, writer) + missing_required = writer.missing_required_output_ids() + if missing_required: + reason = "Case completed without all required outputs" + writer.record_failure(reason) + status = "failed" if EXECUTION["fail_on_missing_required_output"] else "passed" + records.append( + _case_record( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + status=status, + seed=seed, + started_at=started_at, + writer=writer, + reason=reason, + ).as_dict() + ) + else: + # Record any missing diagnostic outputs without failing the case + writer.record_failure("Case did not produce this declared diagnostic output") + records.append( + _case_record( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + status="passed", + seed=seed, + started_at=started_at, + writer=writer, + ).as_dict() + ) + except UnsupportedCase as exc: + reason = str(exc) or "Case is not supported by this backend" + writer.record_skipped(reason) + records.append( + _case_record( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + status="skipped_unsupported", + seed=seed, + started_at=started_at, + writer=writer, + reason=reason, + ).as_dict() + ) + except Exception as exc: + error_text = traceback.format_exc() + reason = f"{type(exc).__name__}: {exc}" + writer.record_failure(reason) + records.append( + _case_record( + test_id=test_id, + case_id=case_id, + profile_id=profile_id, + status="failed", + seed=seed, + started_at=started_at, + writer=writer, + reason=reason, + traceback_text=error_text, + ).as_dict() + ) + print(error_text, file=sys.stderr) + finally: + shutil.rmtree(temporary_directory, ignore_errors=True) + + statuses = {record["status"] for record in records} + if "failed" in statuses: + return "failed", records, None + if statuses == {"skipped_unsupported"}: + return "skipped_unsupported", records, profile_reason + return "passed", records, None + + +def main() -> int: + args = parse_args() + task = load_task(args.task_file) + started_at = utc_now() + status_path = args.results_dir / str(task["status_relative_path"]) + + try: + task_status, case_records, reason = run_cases(task, args.results_dir) + except Exception as exc: + error_text = traceback.format_exc() + print(error_text, file=sys.stderr) + task_status = "failed" + case_records = [] + reason = f"{type(exc).__name__}: {exc}" + + record = { + "task_id": task["task_id"], + "test_id": task["test_id"], + "profile_id": task["profile_id"], + "device": task["device"], + "status": task_status, + "started_at_utc": started_at, + "ended_at_utc": utc_now(), + "case_records": case_records, + "reason": reason, + "observations_relative_path": task["observations_relative_path"], + } + status_path.parent.mkdir(parents=True, exist_ok=True) + status_path.write_text( + json.dumps(record, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + return 0 if task_status in {"passed", "skipped_unsupported"} else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/subprocess_runner.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/subprocess_runner.py new file mode 100644 index 00000000..30852001 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/orchestrator/subprocess_runner.py @@ -0,0 +1,148 @@ +"""Launch one execution task in a clean Python subprocess.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, Mapping + +from config.suite_config import SUBPROCESS_ENVIRONMENT +from pytorch_extended_tests.orchestrator.execution_plan import ExecutionTask +from pytorch_extended_tests.results.observation import utc_now + + +def write_task_file(task: ExecutionTask, results_root: Path) -> Path: + task_directory = results_root / Path(task.status_relative_path).parent + task_directory.mkdir(parents=True, exist_ok=True) + path = task_directory / "task.json" + path.write_text( + json.dumps(task.as_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return path + + +def run_task_subprocess( + task: ExecutionTask, + *, + results_root: Path, + timeout_seconds: int, +) -> dict[str, Any]: + """Run a task and return its task-status record.""" + + task_file = write_task_file(task, results_root) + command = [ + sys.executable, + "-m", + "pytorch_extended_tests.orchestrator.run_test_file", + "--task-file", + str(task_file), + "--results-dir", + str(results_root), + ] + environment = os.environ.copy() + environment.update(SUBPROCESS_ENVIRONMENT) + + started_at = utc_now() + task_directory = task_file.parent + stdout_path = task_directory / "stdout.log" + stderr_path = task_directory / "stderr.log" + + try: + completed = subprocess.run( + command, + cwd=Path(__file__).resolve().parents[3], + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + if isinstance(stdout, bytes): + stdout = stdout.decode("utf-8", errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode("utf-8", errors="replace") + stdout_path.write_text(stdout, encoding="utf-8") + stderr_path.write_text(stderr, encoding="utf-8") + record = _synthetic_task_record( + task, + status="timed_out", + started_at=started_at, + reason=f"Task exceeded the {timeout_seconds} second timeout", + return_code=None, + ) + _write_json(results_root / task.status_relative_path, record) + print(stdout, end="") + print(stderr, end="", file=sys.stderr) + return record + + stdout_path.write_text(completed.stdout, encoding="utf-8") + stderr_path.write_text(completed.stderr, encoding="utf-8") + print(completed.stdout, end="") + print(completed.stderr, end="", file=sys.stderr) + + status_path = results_root / task.status_relative_path + if status_path.is_file(): + try: + record = json.loads(status_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + record = _synthetic_task_record( + task, + status="failed", + started_at=started_at, + reason="Task status file was not valid JSON", + return_code=completed.returncode, + ) + else: + record = _synthetic_task_record( + task, + status="failed", + started_at=started_at, + reason="Task process did not write its status file", + return_code=completed.returncode, + ) + + if completed.returncode != 0 and record.get("status") == "passed": + record["status"] = "failed" + record["reason"] = f"Task process returned {completed.returncode} after reporting success" + record["return_code"] = completed.returncode + _write_json(status_path, record) + return record + + +def _synthetic_task_record( + task: ExecutionTask, + *, + status: str, + started_at: str, + reason: str, + return_code: int | None, +) -> dict[str, Any]: + return { + "task_id": task.task_id, + "test_id": task.test_id, + "profile_id": task.profile_id, + "device": task.device, + "status": status, + "started_at_utc": started_at, + "ended_at_utc": utc_now(), + "case_records": [], + "reason": reason, + "return_code": return_code, + "observations_relative_path": task.observations_relative_path, + } + + +def _write_json(path: Path, value: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/precision_settings.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/precision_settings.py new file mode 100644 index 00000000..83ff3786 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/precision_settings.py @@ -0,0 +1,69 @@ +"""Apply and inspect the float32 precision settings used by the suite.""" + +from __future__ import annotations + +from typing import Any + + +def apply_float32_precision(*, allow_tf32: bool, matmul_precision: str) -> None: + """Apply the configured matmul and convolution precision without mixing APIs.""" + + import torch + + # This public control updates the matmul precision state used by PyTorch 2.9 + # Do not also touch cuda.matmul.allow_tf32 as old and new TF32 APIs must not be mixed + torch.set_float32_matmul_precision(str(matmul_precision)) + + if not hasattr(torch.backends, "cudnn"): + return + + cudnn = torch.backends.cudnn + if hasattr(cudnn, "conv") and hasattr(cudnn.conv, "fp32_precision"): + # PyTorch 2.9 exposes the operator-level cuDNN precision setting + cudnn.conv.fp32_precision = "tf32" if allow_tf32 else "ieee" + elif hasattr(cudnn, "allow_tf32"): + # Keep the legacy fallback for older builds which may still be useful as references + cudnn.allow_tf32 = bool(allow_tf32) + + +def _optional_precision_value(owner: Any, attribute: str) -> str | None: + if owner is None or not hasattr(owner, attribute): + return None + return str(getattr(owner, attribute)) + + +def float32_precision_record() -> dict[str, Any]: + """Return the precision settings exposed by the active PyTorch release.""" + + import torch + + cuda_matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None) + cudnn = getattr(torch.backends, "cudnn", None) + cudnn_conv = getattr(cudnn, "conv", None) if cudnn is not None else None + + record: dict[str, Any] = { + "float32_matmul_precision": torch.get_float32_matmul_precision(), + "global_fp32_precision": _optional_precision_value( + torch.backends, "fp32_precision" + ), + "cuda_matmul_fp32_precision": _optional_precision_value( + cuda_matmul, "fp32_precision" + ), + "cudnn_backend_fp32_precision": _optional_precision_value( + cudnn, "fp32_precision" + ), + "cudnn_convolution_precision": None, + "cudnn_precision_api": None, + } + if cudnn is None: + return record + + if cudnn_conv is not None and hasattr(cudnn_conv, "fp32_precision"): + record["cudnn_convolution_precision"] = str(cudnn_conv.fp32_precision) + record["cudnn_precision_api"] = "fp32_precision" + elif hasattr(cudnn, "allow_tf32"): + record["cudnn_convolution_precision"] = ( + "tf32" if bool(cudnn.allow_tf32) else "ieee" + ) + record["cudnn_precision_api"] = "allow_tf32" + return record diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/__init__.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/__init__.py new file mode 100644 index 00000000..c2e02115 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/__init__.py @@ -0,0 +1,12 @@ +"""Result records and artifact storage for the test suite.""" + +from .artifact_writer import CaseObservationWriter +from .observation import CaseExecutionRecord, ObservationRecord +from .result_bundle import ResultBundle + +__all__ = [ + "CaseExecutionRecord", + "CaseObservationWriter", + "ObservationRecord", + "ResultBundle", +] diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/artifact_writer.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/artifact_writer.py new file mode 100644 index 00000000..9e17fd73 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/artifact_writer.py @@ -0,0 +1,227 @@ +"""Record case outputs and move tensor payloads into the artifact tree.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import asdict, is_dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from config.suite_config import OUTPUT_CAPTURE +from config.test_catalogue import TestSpec +from pytorch_extended_tests.results.observation import ObservationRecord, json_safe +from pytorch_extended_tests.results.tensor_storage import is_tensor_like, write_tensor_artifact + + +_SAFE_COMPONENT = re.compile(r"[^A-Za-z0-9_.-]+") + + +def safe_component(value: str) -> str: + """Make an identifier safe to use as one path component.""" + + cleaned = _SAFE_COMPONENT.sub("_", value).strip("._") + return cleaned or "unnamed" + + +class CaseObservationWriter: + """Validate and store all declared outputs for one case.""" + + def __init__( + self, + *, + results_root: Path, + observations_path: Path, + test_spec: TestSpec, + case_id: str, + profile_id: str, + seed: int, + ) -> None: + self._results_root = results_root + self._observations_path = observations_path + self._test_spec = test_spec + self._case_id = case_id + self._profile_id = profile_id + self._seed = seed + self._output_specs = {item.output_id: item for item in test_spec.outputs} + self._produced_output_ids: set[str] = set() + self._artifact_directory = ( + results_root + / "artifacts" + / safe_component(test_spec.level) + / safe_component(test_spec.test_id) + / safe_component(profile_id) + / safe_component(case_id) + ) + + @property + def produced_output_ids(self) -> tuple[str, ...]: + return tuple( + item.output_id + for item in self._test_spec.outputs + if item.output_id in self._produced_output_ids + ) + + def record( + self, + output_id: str, + value: Any, + *, + coordinates: Mapping[str, Any] | None = None, + ) -> None: + """Store one output declared for this test.""" + + try: + output_spec = self._output_specs[output_id] + except KeyError as exc: + raise KeyError( + f"{self._test_spec.test_id} did not declare output {output_id!r}" + ) from exc + if output_id in self._produced_output_ids: + raise ValueError(f"Output {output_id!r} was recorded more than once") + + self._validate_top_level_kind(output_spec.kind, value) + payload = self._store_value(value, path_parts=(output_id,)) + self._write_record( + ObservationRecord( + test_id=self._test_spec.test_id, + case_id=self._case_id, + profile_id=self._profile_id, + output_id=output_id, + kind=output_spec.kind, + importance=output_spec.importance, + status="produced", + seed=self._seed, + payload=payload, + coordinates=coordinates, + ) + ) + self._produced_output_ids.add(output_id) + + def record_skipped(self, reason: str) -> None: + """Record every declared output as unsupported.""" + + for output_spec in self._test_spec.outputs: + if output_spec.output_id in self._produced_output_ids: + continue + self._write_record( + ObservationRecord( + test_id=self._test_spec.test_id, + case_id=self._case_id, + profile_id=self._profile_id, + output_id=output_spec.output_id, + kind=output_spec.kind, + importance=output_spec.importance, + status="skipped_unsupported", + seed=self._seed, + reason=reason, + ) + ) + + def record_failure(self, reason: str) -> None: + """Record outputs which were not produced after a case failure.""" + + for output_spec in self._test_spec.outputs: + if output_spec.output_id in self._produced_output_ids: + continue + self._write_record( + ObservationRecord( + test_id=self._test_spec.test_id, + case_id=self._case_id, + profile_id=self._profile_id, + output_id=output_spec.output_id, + kind=output_spec.kind, + importance=output_spec.importance, + status="failed_to_produce", + seed=self._seed, + reason=reason, + ) + ) + + def missing_required_output_ids(self) -> tuple[str, ...]: + return tuple( + item.output_id + for item in self._test_spec.outputs + if item.importance == "required" and item.output_id not in self._produced_output_ids + ) + + def _validate_top_level_kind(self, kind: str, value: Any) -> None: + if kind == "scalar" and not self._is_scalar(value): + raise TypeError("Scalar outputs must contain one scalar value") + if kind == "tensor" and not is_tensor_like(value): + raise TypeError("Tensor outputs must contain one tensor or NumPy array") + if kind == "exact_record" and self._contains_tensor(value): + raise TypeError("Exact records cannot contain tensor payloads") + if kind in {"tensor_map", "invariant_bundle"} and not isinstance(value, Mapping): + raise TypeError(f"{kind} outputs must contain a mapping") + if kind in {"tensor_map", "invariant_bundle"} and not self._contains_tensor(value): + raise TypeError(f"{kind} outputs must contain at least one tensor") + if kind == "series" and not self._is_series(value): + raise TypeError("Series outputs must be a one-dimensional tensor or scalar sequence") + + def _store_value(self, value: Any, *, path_parts: tuple[str, ...]) -> Any: + if is_tensor_like(value): + relative_path = self._tensor_relative_path(path_parts) + return write_tensor_artifact( + value, + destination=self._results_root / relative_path, + relative_path=relative_path, + ) + if is_dataclass(value) and not isinstance(value, type): + return self._store_value(asdict(value), path_parts=path_parts) + if isinstance(value, np.generic): + return json_safe(value.item()) + if isinstance(value, Mapping): + stored: dict[str, Any] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise TypeError("Artifact mappings must use string keys") + stored[key] = self._store_value(item, path_parts=(*path_parts, key)) + return stored + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + if len(value) > OUTPUT_CAPTURE["maximum_inline_series_length"]: + raise ValueError("Inline output sequence exceeds the configured maximum length") + return [ + self._store_value(item, path_parts=(*path_parts, f"item_{index:06d}")) + for index, item in enumerate(value) + ] + return json_safe(value) + + def _tensor_relative_path(self, path_parts: tuple[str, ...]) -> Path: + logical_name = "__".join(path_parts) + readable_name = "__".join(safe_component(part) for part in path_parts)[:140] + suffix = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[:12] + filename = f"{readable_name}__{suffix}.bin" + return self._artifact_directory.relative_to(self._results_root) / filename + + def _write_record(self, record: ObservationRecord) -> None: + self._observations_path.parent.mkdir(parents=True, exist_ok=True) + with self._observations_path.open("a", encoding="utf-8", newline="\n") as output: + json.dump(record.as_dict(), output, sort_keys=True, allow_nan=False) + output.write("\n") + + @staticmethod + def _is_scalar(value: Any) -> bool: + return isinstance(value, (bool, int, float, np.generic)) + + @staticmethod + def _contains_tensor(value: Any) -> bool: + if is_tensor_like(value): + return True + if isinstance(value, Mapping): + return any(CaseObservationWriter._contains_tensor(item) for item in value.values()) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return any(CaseObservationWriter._contains_tensor(item) for item in value) + return False + + @staticmethod + def _is_series(value: Any) -> bool: + if is_tensor_like(value): + return getattr(value, "ndim", None) == 1 + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return all(CaseObservationWriter._is_scalar(item) for item in value) + return False diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/level_0_summary.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/level_0_summary.py new file mode 100644 index 00000000..d4fc3bbc --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/level_0_summary.py @@ -0,0 +1,114 @@ +"""Write the concise CSV produced by the Level 0 demonstration run.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any, Iterable, Mapping + +from config.suite_config import LEVEL_0_DEMOS + + +CSV_FIELDS = ( + "test_id", + "case_id", + "profile_id", + "status", + "reason", + "model_type", + "optimiser", + "training_steps", + "device", + "dtype", + "sample_count", + "class_count", + "initial_loss", + "final_loss", + "loss_change", + "initial_accuracy", + "final_accuracy", + "prediction_changes", + "initial_predictions", + "final_predictions", + "initial_logits_mean", + "initial_logits_standard_deviation", + "initial_logits_maximum_absolute", + "final_logits_mean", + "final_logits_standard_deviation", + "final_logits_maximum_absolute", + "first_gradient_l2", + "initial_parameter_l2", + "final_parameter_l2", + "activation_count", + "activation_mean_absolute", + "activation_maximum_absolute", + "activation_names", +) + + +def _format_value(value: Any) -> Any: + if isinstance(value, list): + return " ".join(str(item) for item in value) + if value is None: + return "" + return value + + +def _summary_payloads(path: Path) -> dict[tuple[str, str, str], Mapping[str, Any]]: + output: dict[tuple[str, str, str], Mapping[str, Any]] = {} + if not path.is_file(): + return output + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + record = json.loads(line) + if record.get("output_id") != "summary" or record.get("status") != "produced": + continue + payload = record.get("payload") + if isinstance(payload, Mapping): + key = ( + str(record.get("test_id")), + str(record.get("case_id")), + str(record.get("profile_id")), + ) + output[key] = payload + return output + + +def write_level_0_summary( + results_root: Path, + task_records: Iterable[Mapping[str, Any]], +) -> Path | None: + """Write one row per Level 0 example/profile when Level 0 was selected.""" + + level_0_tasks = [ + task for task in task_records if task.get("test_id") == "demo.model_workloads" + ] + if not level_0_tasks: + return None + + summaries = _summary_payloads(results_root / "observations.jsonl") + rows: list[dict[str, Any]] = [] + for task in level_0_tasks: + test_id = str(task.get("test_id")) + profile_id = str(task.get("profile_id")) + for case in task.get("case_records", []): + case_id = str(case.get("case_id")) + row: dict[str, Any] = { + "test_id": test_id, + "case_id": case_id, + "profile_id": profile_id, + "status": case.get("status", "unknown"), + "reason": case.get("reason") or "", + } + payload = summaries.get((test_id, case_id, profile_id), {}) + row.update(payload) + rows.append({field: _format_value(row.get(field)) for field in CSV_FIELDS}) + + path = results_root / str(LEVEL_0_DEMOS["summary_filename"]) + with path.open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter(output, fieldnames=CSV_FIELDS, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + return path diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/observation.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/observation.py new file mode 100644 index 00000000..1a7afdac --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/observation.py @@ -0,0 +1,127 @@ +"""Machine-readable records written by the execution harness.""" + +from __future__ import annotations + +import math +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from typing import Any, Mapping + +import numpy as np + +from config.suite_config import ( + RESULT_FORMAT_VERSION, + SUITE_VERSION, + TEST_CATALOGUE_VERSION, +) + + +OBSERVATION_STATUSES = { + "produced", + "skipped_unsupported", + "failed_to_produce", +} +CASE_STATUSES = {"passed", "failed", "skipped_unsupported"} +TASK_STATUSES = {"passed", "failed", "skipped_unsupported", "timed_out"} + + +def utc_now() -> str: + """Return an ISO timestamp in UTC.""" + + return datetime.now(timezone.utc).isoformat() + + +def json_safe(value: Any) -> Any: + """Convert ordinary Python and NumPy values into strict JSON data.""" + + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + if math.isfinite(value): + return value + if math.isnan(value): + label = "nan" + elif value > 0: + label = "positive_infinity" + else: + label = "negative_infinity" + return {"value_type": "special_float", "value": label} + if isinstance(value, np.generic): + return json_safe(value.item()) + if isinstance(value, Path): + return value.as_posix() + if isinstance(value, Enum): + return json_safe(value.value) + if isinstance(value, Mapping): + converted: dict[str, Any] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise TypeError("JSON record mappings must use string keys") + converted[key] = json_safe(item) + return converted + if isinstance(value, (list, tuple)): + return [json_safe(item) for item in value] + raise TypeError(f"Value cannot be represented in a JSON record: {type(value)!r}") + + +@dataclass(frozen=True, slots=True) +class ObservationRecord: + """One declared case output and its stored payload.""" + + test_id: str + case_id: str + profile_id: str + output_id: str + kind: str + importance: str + status: str + seed: int + payload: Any | None = None + coordinates: Mapping[str, Any] | None = None + reason: str | None = None + created_at_utc: str = "" + + def __post_init__(self) -> None: + if self.status not in OBSERVATION_STATUSES: + raise ValueError(f"Unknown observation status: {self.status}") + + def as_dict(self) -> dict[str, Any]: + """Return the record with the shared format metadata attached.""" + + data = asdict(self) + if not data["created_at_utc"]: + data["created_at_utc"] = utc_now() + data.update( + { + "result_format_version": RESULT_FORMAT_VERSION, + "suite_version": SUITE_VERSION, + "test_catalogue_version": TEST_CATALOGUE_VERSION, + } + ) + return json_safe(data) + + +@dataclass(frozen=True, slots=True) +class CaseExecutionRecord: + """Execution status for one test case and profile.""" + + test_id: str + case_id: str + profile_id: str + status: str + seed: int + started_at_utc: str + ended_at_utc: str + produced_output_ids: tuple[str, ...] = () + missing_required_output_ids: tuple[str, ...] = () + reason: str | None = None + traceback: str | None = None + + def __post_init__(self) -> None: + if self.status not in CASE_STATUSES: + raise ValueError(f"Unknown case status: {self.status}") + + def as_dict(self) -> dict[str, Any]: + return json_safe(asdict(self)) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/result_bundle.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/result_bundle.py new file mode 100644 index 00000000..c2da1baf --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/result_bundle.py @@ -0,0 +1,129 @@ +"""Create and finalise the result directory collected by CI.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +from pathlib import Path +from typing import Any, Iterable, Mapping + +from config.suite_config import ( + RESULT_FORMAT_VERSION, + ROOT_SEED, + SUITE_NAME, + SUITE_VERSION, + TEST_CATALOGUE_VERSION, +) +from pytorch_extended_tests.results.level_0_summary import write_level_0_summary +from pytorch_extended_tests.results.observation import utc_now + + +class ResultBundle: + """Own the top-level files for one suite invocation.""" + + def __init__(self, results_root: Path, *, remove_existing: bool) -> None: + self.results_root = results_root + self.tasks_root = results_root / ".work" / "tasks" + self.started_at_utc = utc_now() + + if remove_existing and results_root.exists(): + shutil.rmtree(results_root) + results_root.mkdir(parents=True, exist_ok=True) + self.tasks_root.mkdir(parents=True, exist_ok=True) + (results_root / "artifacts").mkdir(parents=True, exist_ok=True) + + @staticmethod + def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def write_initial_manifest( + self, + *, + dataset_manifest_path: Path, + device: str, + profile_ids: Iterable[str], + planned_task_count: int, + ) -> None: + manifest = { + "suite_name": SUITE_NAME, + "suite_version": SUITE_VERSION, + "result_format_version": RESULT_FORMAT_VERSION, + "test_catalogue_version": TEST_CATALOGUE_VERSION, + "root_seed": ROOT_SEED, + "dataset_manifest_sha256": self.hash_file(dataset_manifest_path), + "device": device, + "profile_ids": list(profile_ids), + "planned_task_count": planned_task_count, + "started_at_utc": self.started_at_utc, + "ended_at_utc": None, + "overall_execution_status": "running", + "counts": {}, + } + self._write_json(self.results_root / "run_manifest.json", manifest) + + def finalise( + self, + *, + task_records: Iterable[Mapping[str, Any]], + overall_status: str, + ) -> None: + records = list(task_records) + self._consolidate_observations(records) + write_level_0_summary(self.results_root, records) + final_records = [self._final_task_record(record) for record in records] + self._write_json(self.results_root / "test_status.json", {"tasks": final_records}) + + manifest_path = self.results_root / "run_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["ended_at_utc"] = utc_now() + manifest["overall_execution_status"] = overall_status + manifest["counts"] = self._count_statuses(records) + self._write_json(manifest_path, manifest) + + # Keep temporary files out of the artifact uploaded by CI + shutil.rmtree(self.results_root / ".work", ignore_errors=True) + + def _consolidate_observations(self, task_records: list[Mapping[str, Any]]) -> None: + output_path = self.results_root / "observations.jsonl" + with output_path.open("w", encoding="utf-8", newline="\n") as output: + for task in task_records: + relative_path = task.get("observations_relative_path") + if not isinstance(relative_path, str): + continue + source_path = self.results_root / relative_path + if not source_path.is_file(): + continue + with source_path.open("r", encoding="utf-8") as source: + shutil.copyfileobj(source, output) + + @staticmethod + def _final_task_record(record: Mapping[str, Any]) -> dict[str, Any]: + # The per-task observation files are removed after consolidation + # Do not leave paths in the final status file which no longer exist + final_record = dict(record) + final_record.pop("observations_relative_path", None) + final_record["observations_file"] = "observations.jsonl" + return final_record + + @staticmethod + def _count_statuses(records: list[Mapping[str, Any]]) -> dict[str, int]: + counts: dict[str, int] = {} + for record in records: + status = str(record.get("status", "unknown")) + counts[status] = counts.get(status, 0) + 1 + return counts + + @staticmethod + def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.tmp") + temporary_path.write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + temporary_path.replace(path) diff --git a/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/tensor_storage.py b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/tensor_storage.py new file mode 100644 index 00000000..b9258918 --- /dev/null +++ b/pytorch/pytorch_extended_tests/src/pytorch_extended_tests/results/tensor_storage.py @@ -0,0 +1,210 @@ +"""Lossless tensor storage used by the result artifact writer.""" + +from __future__ import annotations + +import hashlib +import os +import tempfile +from pathlib import Path +from typing import Any + +import numpy as np + + +RAW_TENSOR_FORMAT = "raw_little_endian_v1" + + +def is_tensor_like(value: Any) -> bool: + """Return whether a value is a PyTorch tensor or NumPy array.""" + + if isinstance(value, np.ndarray): + return True + try: + import torch + except ImportError: + return False + return isinstance(value, torch.Tensor) + + +def _normalise_numpy_array(array: np.ndarray) -> tuple[np.ndarray, str, str]: + if array.dtype.hasobject: + raise TypeError("Object arrays cannot be stored as tensor artifacts") + + contiguous = np.ascontiguousarray(array) + logical_dtype = contiguous.dtype.name + storage_dtype = contiguous.dtype.newbyteorder("<") + little_endian = contiguous.astype(storage_dtype, copy=False) + return little_endian, logical_dtype, little_endian.dtype.str + + +def _normalise_torch_tensor(value: Any) -> tuple[np.ndarray, str, str, dict[str, Any]]: + import torch + + tensor = value.detach() + source = { + "source_device": str(tensor.device), + "source_strides": list(tensor.stride()), + "source_contiguous": tensor.is_contiguous(), + "requires_grad": bool(value.requires_grad), + } + cpu_tensor = tensor.to(device="cpu").contiguous() + logical_dtype = str(cpu_tensor.dtype).removeprefix("torch.") + + if cpu_tensor.dtype == torch.bfloat16: + # NumPy has no portable bfloat16 dtype + # Store the exact two-byte representation and keep the logical dtype separately + array = cpu_tensor.view(torch.uint16).numpy() + storage_dtype = np.dtype(" np.ndarray: + """Copy a tensor to CPU before using NumPy for summary statistics.""" + + try: + import torch + except ImportError: + torch = None + + if torch is not None and isinstance(value, torch.Tensor): + tensor = value.detach() + + # NumPy cannot read CUDA tensors directly + if tensor.device.type != "cpu": + tensor = tensor.cpu() + + # NumPy generally has no native bfloat16 representation + # Float32 is sufficient here because this copy is only used for counts + if tensor.dtype == torch.bfloat16: + tensor = tensor.to(torch.float32) + + return tensor.numpy() + + return np.asarray(value) + + +def _exceptional_value_counts(value: Any, numel: int) -> dict[str, int | None]: + try: + import torch + except ImportError: + torch = None + + if torch is not None and isinstance(value, torch.Tensor): + tensor = value.detach().to(device="cpu") + if tensor.dtype == torch.bfloat16: + tensor = tensor.to(dtype=torch.float32) + if tensor.is_floating_point() or tensor.is_complex(): + finite_count = int(torch.isfinite(tensor).sum().item()) + nan_count = int(torch.isnan(tensor).sum().item()) + infinity_count = int(torch.isinf(tensor).sum().item()) + if tensor.is_complex(): + positive_infinity_count = None + negative_infinity_count = None + else: + positive_infinity_count = int(torch.isposinf(tensor).sum().item()) + negative_infinity_count = int(torch.isneginf(tensor).sum().item()) + return { + "finite_count": finite_count, + "nan_count": nan_count, + "infinity_count": infinity_count, + "positive_infinity_count": positive_infinity_count, + "negative_infinity_count": negative_infinity_count, + } + + array = _as_numpy_for_statistics(value) + if np.issubdtype(array.dtype, np.inexact): + finite = np.isfinite(array) + nan = np.isnan(array) + infinity = np.isinf(array) + if np.issubdtype(array.dtype, np.complexfloating): + positive_infinity_count = None + negative_infinity_count = None + else: + positive_infinity_count = int(np.isposinf(array).sum()) + negative_infinity_count = int(np.isneginf(array).sum()) + return { + "finite_count": int(finite.sum()), + "nan_count": int(nan.sum()), + "infinity_count": int(infinity.sum()), + "positive_infinity_count": positive_infinity_count, + "negative_infinity_count": negative_infinity_count, + } + + return { + "finite_count": numel, + "nan_count": 0, + "infinity_count": 0, + "positive_infinity_count": 0, + "negative_infinity_count": 0, + } + + +def _atomic_write_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + file_descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary_path = Path(temporary_name) + try: + with os.fdopen(file_descriptor, "wb") as output: + output.write(payload) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary_path, path) + finally: + temporary_path.unlink(missing_ok=True) + + +def write_tensor_artifact( + value: Any, + *, + destination: Path, + relative_path: Path, +) -> dict[str, Any]: + """Write one tensor as canonical bytes and return its descriptor.""" + + try: + import torch + except ImportError: + torch = None + + if torch is not None and isinstance(value, torch.Tensor): + array, logical_dtype, storage_dtype, source = _normalise_torch_tensor(value) + shape = list(value.shape) + elif isinstance(value, np.ndarray): + array, logical_dtype, storage_dtype = _normalise_numpy_array(value) + source = { + "source_device": "cpu", + "source_strides_bytes": list(value.strides), + "source_contiguous": bool(value.flags.c_contiguous), + "requires_grad": False, + } + shape = list(value.shape) + else: + raise TypeError(f"Expected a tensor or NumPy array, got {type(value)!r}") + + payload = array.tobytes(order="C") + checksum = hashlib.sha256(payload).hexdigest() + _atomic_write_bytes(destination, payload) + numel = int(np.prod(shape, dtype=np.int64)) if shape else 1 + + return { + "artifact_type": "tensor", + "storage_format": RAW_TENSOR_FORMAT, + "relative_path": relative_path.as_posix(), + "sha256": checksum, + "byte_length": len(payload), + "logical_dtype": logical_dtype, + "storage_dtype": storage_dtype, + "byte_order": "little", + "shape": shape, + "numel": numel, + **source, + **_exceptional_value_counts(value, numel), + } diff --git a/pytorch/pytorch_extended_tests/tools/README.md b/pytorch/pytorch_extended_tests/tools/README.md new file mode 100644 index 00000000..ab88255d --- /dev/null +++ b/pytorch/pytorch_extended_tests/tools/README.md @@ -0,0 +1,32 @@ +# Tools + +These are the small maintenance tools used while running and checking the test suite + +The manual repeatability and comparison scripts live only in `manual_comparison_stuff/` +They are deliberately kept out of this folder because CI does not run them + +## `validate_setup.py` + +Checks the central config, catalogue, selected datasets, PyTorch device and case-module imports before a run starts + +```bash +python tools/validate_setup.py +``` + +For a CPU run: + +```bash +python tools/validate_setup.py --device cpu +``` + +It accepts the same basic profile, level and test filters as the suite orchestrator + +## `inspect_result_bundle.py` + +Checks a completed raw result bundle, including every tensor artefact and its SHA-256 hash + +```bash +python tools/inspect_result_bundle.py /tmp/ci_benchmarks/pytorch +``` + +Use `--json` when a machine-readable validation result is more useful diff --git a/pytorch/pytorch_extended_tests/tools/inspect_result_bundle.py b/pytorch/pytorch_extended_tests/tools/inspect_result_bundle.py new file mode 100644 index 00000000..d564739d --- /dev/null +++ b/pytorch/pytorch_extended_tests/tools/inspect_result_bundle.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Inspect and validate one raw result bundle written by the suite.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPOSITORY_ROOT / "src" +for path in (REPOSITORY_ROOT, SRC_ROOT): + value = str(path) + if value not in sys.path: + sys.path.insert(0, value) + +from config.suite_config import RESULTS_DIR # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("result_directory", nargs="?", type=Path, default=RESULTS_DIR) + parser.add_argument( + "--skip-artifact-hashes", + action="store_true", + help="Check that artifacts exist but do not recalculate their SHA-256 hashes", + ) + parser.add_argument( + "--json", + action="store_true", + help="Print the inspection summary as JSON", + ) + return parser.parse_args() + + +def read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise RuntimeError(f"Required result file is missing: {path}") from exc + except json.JSONDecodeError as exc: + raise RuntimeError(f"Result file is not valid JSON: {path}") from exc + + +def read_json_lines(path: Path) -> list[dict[str, Any]]: + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError as exc: + raise RuntimeError(f"Required result file is missing: {path}") from exc + + records: list[dict[str, Any]] = [] + for line_number, line in enumerate(lines, start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Invalid JSON on {path}:{line_number}") from exc + if not isinstance(value, dict): + raise RuntimeError(f"Observation on {path}:{line_number} is not an object") + records.append(value) + return records + + +def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def iter_tensor_descriptors(value: Any) -> Iterable[dict[str, Any]]: + if isinstance(value, dict): + if value.get("artifact_type") == "tensor": + yield value + return + for item in value.values(): + yield from iter_tensor_descriptors(item) + elif isinstance(value, list): + for item in value: + yield from iter_tensor_descriptors(item) + + +def inspect_artifacts( + result_directory: Path, + observations: list[dict[str, Any]], + *, + verify_hashes: bool, +) -> dict[str, Any]: + descriptor_count = 0 + total_bytes = 0 + seen_paths: set[str] = set() + + for observation in observations: + for descriptor in iter_tensor_descriptors(observation.get("payload")): + descriptor_count += 1 + relative_path = descriptor.get("relative_path") + if not isinstance(relative_path, str): + raise RuntimeError("Tensor descriptor has no relative_path") + if relative_path in seen_paths: + raise RuntimeError(f"Tensor artifact is referenced more than once: {relative_path}") + seen_paths.add(relative_path) + + path = result_directory / relative_path + if not path.is_file(): + raise RuntimeError(f"Tensor artifact is missing: {path}") + expected_length = descriptor.get("byte_length") + if path.stat().st_size != expected_length: + raise RuntimeError(f"Tensor artifact size does not match its descriptor: {path}") + total_bytes += path.stat().st_size + + if verify_hashes: + expected_hash = descriptor.get("sha256") + if hash_file(path) != expected_hash: + raise RuntimeError(f"Tensor artifact hash does not match: {path}") + + artifact_files = { + path.relative_to(result_directory).as_posix() + for path in (result_directory / "artifacts").rglob("*.bin") + } + unreferenced = sorted(artifact_files - seen_paths) + if unreferenced: + raise RuntimeError( + "The bundle contains unreferenced tensor artifacts\n" + + "\n".join(unreferenced[:20]) + ) + + return { + "tensor_descriptor_count": descriptor_count, + "tensor_artifact_count": len(artifact_files), + "tensor_artifact_bytes": total_bytes, + "hashes_verified": verify_hashes, + } + + +def main() -> int: + args = parse_args() + result_directory = args.result_directory.resolve() + manifest = read_json(result_directory / "run_manifest.json") + status = read_json(result_directory / "test_status.json") + observations = read_json_lines(result_directory / "observations.jsonl") + + tasks = status.get("tasks") if isinstance(status, dict) else None + if not isinstance(tasks, list): + raise RuntimeError("test_status.json does not contain a tasks list") + + task_statuses = Counter(str(task.get("status", "unknown")) for task in tasks) + observation_statuses = Counter( + str(observation.get("status", "unknown")) for observation in observations + ) + output_kinds = Counter(str(observation.get("kind", "unknown")) for observation in observations) + artifact_summary = inspect_artifacts( + result_directory, + observations, + verify_hashes=not args.skip_artifact_hashes, + ) + + level_0_tasks = [task for task in tasks if task.get("test_id") == "demo.model_workloads"] + level_0_summary_rows = 0 + if level_0_tasks: + summary_path = result_directory / "level_0_summary.csv" + if not summary_path.is_file(): + raise RuntimeError("Level 0 ran but level_0_summary.csv is missing") + with summary_path.open("r", encoding="utf-8", newline="") as source: + level_0_summary_rows = sum(1 for _ in csv.DictReader(source)) + expected_rows = sum(len(task.get("case_records", [])) for task in level_0_tasks) + if level_0_summary_rows != expected_rows: + raise RuntimeError( + "level_0_summary.csv row count does not match the Level 0 case count" + ) + + summary = { + "status": "valid", + "result_directory": result_directory.as_posix(), + "suite_name": manifest.get("suite_name"), + "suite_version": manifest.get("suite_version"), + "overall_execution_status": manifest.get("overall_execution_status"), + "planned_task_count": manifest.get("planned_task_count"), + "task_count": len(tasks), + "task_statuses": dict(sorted(task_statuses.items())), + "observation_count": len(observations), + "observation_statuses": dict(sorted(observation_statuses.items())), + "output_kinds": dict(sorted(output_kinds.items())), + "level_0_summary_rows": level_0_summary_rows, + **artifact_summary, + } + + if args.json: + print(json.dumps(summary, indent=2, sort_keys=True)) + else: + print("Result bundle is valid") + print(f"Directory: {result_directory}") + print(f"Execution status: {summary['overall_execution_status']}") + print(f"Tasks: {len(tasks)} {dict(task_statuses)}") + print(f"Observations: {len(observations)} {dict(observation_statuses)}") + print(f"Tensor artifacts: {artifact_summary['tensor_artifact_count']}") + print(f"Tensor bytes: {artifact_summary['tensor_artifact_bytes']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pytorch/pytorch_extended_tests/tools/validate_setup.py b/pytorch/pytorch_extended_tests/tools/validate_setup.py new file mode 100644 index 00000000..2eb3aff0 --- /dev/null +++ b/pytorch/pytorch_extended_tests/tools/validate_setup.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Validate the repository, datasets and selected execution plan.""" + +from __future__ import annotations + +import argparse +import importlib +import json +import os +import sys +from pathlib import Path +from typing import Any + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPOSITORY_ROOT / "src" +for path in (REPOSITORY_ROOT, SRC_ROOT): + value = str(path) + if value not in sys.path: + sys.path.insert(0, value) + +from config.suite_config import ( # noqa: E402 + ALLOWED_DEVICES, + DEFAULT_DEVICE, + DEVICE_ENVIRONMENT_VARIABLE, + EXECUTION, + EXECUTION_PROFILES, + validate_suite_config, +) +from config.test_catalogue import get_test_spec, validate_test_catalogue # noqa: E402 +from pytorch_extended_tests.datasets.validation import validate_datasets # noqa: E402 +from pytorch_extended_tests.orchestrator.execution_plan import build_execution_plan # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device", choices=ALLOWED_DEVICES) + parser.add_argument("--profiles", nargs="+", choices=tuple(EXECUTION_PROFILES)) + parser.add_argument("--levels", nargs="+") + parser.add_argument("--tests", nargs="+") + parser.add_argument( + "--skip-downloaded-source-checks", + action="store_true", + help="Only validate the prepared files needed by the selected tests", + ) + parser.add_argument( + "--json", + action="store_true", + help="Print the validation summary as JSON", + ) + return parser.parse_args() + + +def resolve_device(value: str | None) -> str: + selected = value or os.environ.get(DEVICE_ENVIRONMENT_VARIABLE) or DEFAULT_DEVICE + if selected not in ALLOWED_DEVICES: + raise ValueError( + f"{DEVICE_ENVIRONMENT_VARIABLE} must be one of {', '.join(ALLOWED_DEVICES)}" + ) + return selected + + +def required_dataset_ids(test_ids: list[str]) -> tuple[str, ...]: + ordered: list[str] = [] + for test_id in test_ids: + for dataset_id in get_test_spec(test_id).dataset_ids: + if dataset_id not in ordered: + ordered.append(dataset_id) + return tuple(ordered) + + +def validate_torch_device(device: str) -> dict[str, Any]: + import torch + + if device == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was selected but torch.cuda.is_available() is false") + return { + "torch_imported": True, + "torch_version": torch.__version__, + "selected_device": device, + "cuda_available": bool(torch.cuda.is_available()), + } + + +def main() -> int: + args = parse_args() + device = resolve_device(args.device) + validate_suite_config() + validate_test_catalogue() + + plan = build_execution_plan( + device=device, + profiles=args.profiles, + levels=args.levels, + test_ids=args.tests, + ) + if not plan: + raise RuntimeError("The selected configuration produced an empty execution plan") + + test_ids = list(dict.fromkeys(task.test_id for task in plan)) + dataset_ids = required_dataset_ids(test_ids) + dataset_manifest_sha256 = validate_datasets( + dataset_ids, + validate_downloaded_sources=( + bool(EXECUTION["validate_downloaded_sources"]) + and not args.skip_downloaded_source_checks + ), + ) + + # Import every selected case module now so CI does not discover a typo halfway through + imported_modules: list[str] = [] + for test_id in test_ids: + spec = get_test_spec(test_id) + module = importlib.import_module(spec.module) + run_case = getattr(module, "run_case", None) + if not callable(run_case): + raise TypeError(f"{spec.module}.run_case is not callable") + imported_modules.append(spec.module) + + summary = { + "status": "passed", + "device": validate_torch_device(device), + "task_count": len(plan), + "test_count": len(test_ids), + "tests": test_ids, + "profiles": list(dict.fromkeys(task.profile_id for task in plan)), + "dataset_ids": list(dataset_ids), + "dataset_manifest_sha256": dataset_manifest_sha256, + "imported_modules": imported_modules, + } + if args.json: + print(json.dumps(summary, indent=2, sort_keys=True)) + else: + print("Setup validation passed") + print(f"Device: {device}") + print(f"Tasks: {len(plan)}") + print(f"Tests: {len(test_ids)}") + print(f"Profiles: {', '.join(summary['profiles'])}") + print(f"Datasets: {', '.join(dataset_ids)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())