From f67d71a3c37f38cea75434ca0db812a88d637ffc Mon Sep 17 00:00:00 2001 From: darnstrom Date: Tue, 28 Jul 2026 22:08:01 +0200 Subject: [PATCH] Add Maros-Meszaros problems to CI --- .github/benchmarks/compare_maros_meszaros.py | 101 ++++++++ .github/benchmarks/export_maros_meszaros.py | 122 +++++++++ .../maros_meszaros_comparison_git.sh | 60 +++++ .github/benchmarks/maros_meszaros_runner.c | 234 ++++++++++++++++++ .github/workflows/ci_benchmark.yml | 214 ++++++++++++++-- interfaces/daqp-julia/test/benchmark.jl | 126 ++++++---- .../test/benchmark_comparison_git.sh | 6 +- 7 files changed, 799 insertions(+), 64 deletions(-) create mode 100644 .github/benchmarks/compare_maros_meszaros.py create mode 100644 .github/benchmarks/export_maros_meszaros.py create mode 100644 .github/benchmarks/maros_meszaros_comparison_git.sh create mode 100644 .github/benchmarks/maros_meszaros_runner.c diff --git a/.github/benchmarks/compare_maros_meszaros.py b/.github/benchmarks/compare_maros_meszaros.py new file mode 100644 index 0000000..9e5ee07 --- /dev/null +++ b/.github/benchmarks/compare_maros_meszaros.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Create an aggregate current-vs-master Maros-Meszaros report.""" + +import argparse +import csv + + +TOLERANCES = ("default", "low", "med", "high") + + +def load(path): + with open(path, newline="", encoding="utf-8") as source: + rows = list(csv.DictReader(source)) + return {(row["problem"], row["tolerance"]): row for row in rows} + + +def percent_change(baseline, current): + if baseline == 0.0: + return "n/a" + return f"{100.0 * (current - baseline) / baseline:+.1f}%" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("baseline") + parser.add_argument("current") + parser.add_argument("report") + parser.add_argument("--threshold", type=float, default=25.0) + args = parser.parse_args() + + baseline = load(args.baseline) + current = load(args.current) + if baseline.keys() != current.keys(): + missing = sorted(baseline.keys() - current.keys()) + new = sorted(current.keys() - baseline.keys()) + raise RuntimeError(f"result sets differ; missing={missing}, new={new}") + + lines = [ + "## Maros–Mészáros dense benchmark", + "", + "Solve time is the sum of DAQP's reported solve time over every attempted " + "problem (best of three identical runs per problem). A problem is counted " + "as solved only when DAQP returns success and the KKT residual checks pass.", + ] + regression = False + + for title, predicate in ( + ("Dense subset", lambda row: True), + ("Positive-definite subset", lambda row: row["posdef"] == "1"), + ): + lines.extend( + [ + "", + f"### {title}", + "", + "| Tolerance | Problems | Base solved | Current solved | Δ solved | " + "Base solve time | Current solve time | Δ time |", + "|:--|--:|--:|--:|--:|--:|--:|--:|", + ] + ) + for tolerance in TOLERANCES: + base_rows = [ + row for (_, setting), row in baseline.items() + if setting == tolerance and predicate(row) + ] + current_rows = [ + row for (_, setting), row in current.items() + if setting == tolerance and predicate(row) + ] + base_solved = sum(int(row["solved"]) for row in base_rows) + current_solved = sum(int(row["solved"]) for row in current_rows) + base_time = sum(float(row["solve_time_s"]) for row in base_rows) + current_time = sum(float(row["solve_time_s"]) for row in current_rows) + time_change = 100.0 * (current_time - base_time) / base_time if base_time else 0.0 + solved_change = current_solved - base_solved + if current_solved < base_solved or time_change > args.threshold: + regression = True + lines.append( + f"| {tolerance} | {len(base_rows)} | {base_solved} | " + f"{current_solved} | {solved_change:+d} | {base_time:.4f} s | " + f"{current_time:.4f} s | {percent_change(base_time, current_time)} |" + ) + + lines.extend( + [ + "", + f"Time regressions are flagged above {args.threshold:g}%; any decrease " + "in the solved count is also a regression.", + "", + "⚠️ Regression detected." if regression else "✅ No regression detected.", + ] + ) + report = "\n".join(lines) + "\n" + with open(args.report, "w", encoding="utf-8") as output: + output.write(report) + print(report, end="") + raise SystemExit(1 if regression else 0) + + +if __name__ == "__main__": + main() diff --git a/.github/benchmarks/export_maros_meszaros.py b/.github/benchmarks/export_maros_meszaros.py new file mode 100644 index 0000000..b1ff7bf --- /dev/null +++ b/.github/benchmarks/export_maros_meszaros.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2022 Stéphane Caron +# Copyright 2023-2024 Inria +# Adapted from qpsolvers/maros_meszaros_qpbenchmark. +"""Export the Maros-Meszaros dense subset to DAQP's native dense layout.""" + +import argparse +import os +import struct + +import numpy as np +import scipy.io as spio +import scipy.sparse as spa +from qpbenchmark.utils import is_posdef + + +DAQP_INF = 1e30 +EQ_TOL = 1e-10 + + +def load_mat(path): + mat = spio.loadmat(path) + P = mat["P"].astype(float).tocsc() + q = mat["q"].T.flatten().astype(float) + A = mat["A"].astype(float).tocsc() + lower = mat["l"].T.flatten().astype(float) + upper = mat["u"].T.flatten().astype(float) + n = int(mat["n"].T.flatten()[0]) + m = int(mat["m"].T.flatten()[0]) + assert A.shape == (m, n) + + A = A.copy() + A.data[A.data > 9e19] = np.inf + A.data[A.data < -9e19] = -np.inf + lower[lower > 9e19] = np.inf + lower[lower < -9e19] = -np.inf + upper[upper > 9e19] = np.inf + upper[upper < -9e19] = -np.inf + + # The MAT files store A = vstack([C, eye(n)]). + return P, q, A[:-n], lower[:-n], upper[:-n], lower[-n:], upper[-n:] + + +def converted_constraint_count(C, lower, upper, box_lower): + """Match MarosMeszaros.count_constraints after its format conversion.""" + equal = upper - lower < EQ_TOL + inequality_rows = np.asarray(np.logical_not(equal)).nonzero() + G = spa.vstack([C[inequality_rows], -C[inequality_rows]], format="csc") + h = np.hstack([upper[inequality_rows], -lower[inequality_rows]]) + finite = h < np.inf + inequalities = G[finite].shape[0] if G.size > 0 else 0 + equalities = C[np.asarray(equal).nonzero()].shape[0] + return inequalities + equalities + box_lower.shape[0] + + +def export_problem(output_dir, name, P, q, C, lower, upper, box_lower, box_upper): + H = np.asarray(P.todense(), dtype=np.float64) + H = 0.5 * (H + H.T) + A = np.asarray(C.todense(), dtype=np.float64) + blower = np.concatenate([box_lower, lower]).astype(np.float64) + bupper = np.concatenate([box_upper, upper]).astype(np.float64) + n = H.shape[0] + m = blower.size + sense = np.zeros(m, dtype=np.int32) + + equal = bupper - blower < EQ_TOL + sense[equal] = 5 # DAQP_ACTIVE | DAQP_IMMUTABLE + blower[equal] = bupper[equal] + blower = np.clip(blower, -DAQP_INF, DAQP_INF) + bupper = np.clip(bupper, -DAQP_INF, DAQP_INF) + + with open(os.path.join(output_dir, f"{name}.bin"), "wb") as output: + output.write(struct.pack(" 1000 or converted_m > 1000: + continue + + export_problem(args.output_dir, name, *problem) + # MarosMeszarosDensePosdef applies this check after `to_dense()`. + index.append( + ( + name, + n, + box_lower.size + C.shape[0], + bool(is_posdef(np.asarray(P.todense()))), + ) + ) + + with open(os.path.join(args.output_dir, "index.txt"), "w", encoding="utf-8") as output: + for name, n, m, posdef in index: + output.write(f"{name} {n} {m} {int(posdef)}\n") + + posdef_count = sum(row[3] for row in index) + print(f"Exported {len(index)} dense problems ({posdef_count} positive definite)") + + +if __name__ == "__main__": + main() diff --git a/.github/benchmarks/maros_meszaros_comparison_git.sh b/.github/benchmarks/maros_meszaros_comparison_git.sh new file mode 100644 index 0000000..4f71ef3 --- /dev/null +++ b/.github/benchmarks/maros_meszaros_comparison_git.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Compare the current DAQP library against a git ref on Maros-Meszaros dense QPs. + +set -euo pipefail + +if [ "$#" -lt 4 ]; then + echo "usage: $0 [threshold] [repeats]" + exit 2 +fi + +CURRENT_BUILD="$(cd "$1" && pwd)" +BASE_REF="$2" +MAROS_DATA="$(cd "$3" && pwd)" +OUTPUT_DIR="$(mkdir -p "$4" && cd "$4" && pwd)" +THRESHOLD="${5:-25}" +REPEATS="${6:-3}" +REPO_ROOT="$(git rev-parse --show-toplevel)" +SCRIPT_DIR="$REPO_ROOT/.github/benchmarks" +EXPORTED_DIR="$OUTPUT_DIR/exported" +CURRENT_LIB="$CURRENT_BUILD/interfaces/daqp-julia/libdaqp.so" + +if [ ! -f "$CURRENT_LIB" ]; then + echo "Current libdaqp.so not found at $CURRENT_LIB" + exit 1 +fi + +python3 "$SCRIPT_DIR/export_maros_meszaros.py" "$MAROS_DATA" "$EXPORTED_DIR" + +cc -O3 -I"$REPO_ROOT/include" "$SCRIPT_DIR/maros_meszaros_runner.c" \ + -L"$(dirname "$CURRENT_LIB")" -ldaqp -lm -o "$OUTPUT_DIR/maros_meszaros_runner" + +echo "Benchmarking current library" +LD_LIBRARY_PATH="$(dirname "$CURRENT_LIB")${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + "$OUTPUT_DIR/maros_meszaros_runner" "$EXPORTED_DIR" \ + "$OUTPUT_DIR/current.csv" "$REPEATS" + +TEMP_REPO="$(mktemp -d)" +trap 'rm -rf "$TEMP_REPO"' EXIT +git clone --no-checkout "$REPO_ROOT" "$TEMP_REPO/daqp_base" >/dev/null 2>&1 +git -C "$TEMP_REPO/daqp_base" checkout --detach "$BASE_REF" >/dev/null +cmake -S "$TEMP_REPO/daqp_base" -B "$TEMP_REPO/daqp_base/build" \ + -DCMAKE_BUILD_TYPE=Release >/dev/null +cmake --build "$TEMP_REPO/daqp_base/build" -- -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)" >/dev/null + +BASE_LIB="$(find "$TEMP_REPO/daqp_base/build" -name libdaqp.so -type f -print -quit)" +if [ -z "$BASE_LIB" ]; then + echo "Baseline build did not produce libdaqp.so" + exit 1 +fi + +echo "Benchmarking baseline library $BASE_REF" +LD_LIBRARY_PATH="$(dirname "$BASE_LIB")${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + "$OUTPUT_DIR/maros_meszaros_runner" "$EXPORTED_DIR" \ + "$OUTPUT_DIR/master.csv" "$REPEATS" + +COMPARE_RESULT=0 +python3 "$SCRIPT_DIR/compare_maros_meszaros.py" \ + "$OUTPUT_DIR/master.csv" "$OUTPUT_DIR/current.csv" \ + "$OUTPUT_DIR/comparison.md" --threshold "$THRESHOLD" || COMPARE_RESULT=$? +exit "$COMPARE_RESULT" diff --git a/.github/benchmarks/maros_meszaros_runner.c b/.github/benchmarks/maros_meszaros_runner.c new file mode 100644 index 0000000..c0425ad --- /dev/null +++ b/.github/benchmarks/maros_meszaros_runner.c @@ -0,0 +1,234 @@ +/* Benchmark DAQP on the exported Maros-Meszaros dense subset. */ +#include +#include +#include +#include + +#include "api.h" +#include "constants.h" + +typedef struct { + int n, m, ms; + double *H, *f, *A, *bupper, *blower; + int *sense; +} Problem; + +typedef struct { + int exitflag, iterations, solved; + double solve_time, primal, dual, gap; +} Result; + +typedef struct { + const char *name; + double solver_tol; + double check_tol; + int use_defaults; +} Tolerance; + +static int read_problem(const char *path, Problem *p) { + FILE *file = fopen(path, "rb"); + if (file == NULL) return 0; + if (fread(&p->n, sizeof(int), 1, file) != 1 || + fread(&p->m, sizeof(int), 1, file) != 1 || + fread(&p->ms, sizeof(int), 1, file) != 1) { + fclose(file); + return 0; + } + + const int n = p->n, m = p->m, general = m - p->ms; + p->H = malloc(sizeof(double) * (size_t)n * n); + p->f = malloc(sizeof(double) * (size_t)n); + p->A = general > 0 ? malloc(sizeof(double) * (size_t)general * n) : NULL; + p->bupper = malloc(sizeof(double) * (size_t)m); + p->blower = malloc(sizeof(double) * (size_t)m); + p->sense = malloc(sizeof(int) * (size_t)m); + + int ok = p->H != NULL && p->f != NULL && p->bupper != NULL && + p->blower != NULL && p->sense != NULL && + (general == 0 || p->A != NULL); + ok = ok && fread(p->H, sizeof(double), (size_t)n * n, file) == (size_t)n * n; + ok = ok && fread(p->f, sizeof(double), (size_t)n, file) == (size_t)n; + if (general > 0) + ok = ok && fread(p->A, sizeof(double), (size_t)general * n, file) == + (size_t)general * n; + ok = ok && fread(p->bupper, sizeof(double), (size_t)m, file) == (size_t)m; + ok = ok && fread(p->blower, sizeof(double), (size_t)m, file) == (size_t)m; + ok = ok && fread(p->sense, sizeof(int), (size_t)m, file) == (size_t)m; + fclose(file); + return ok; +} + +static void free_problem(Problem *p) { + free(p->H); + free(p->f); + free(p->A); + free(p->bupper); + free(p->blower); + free(p->sense); +} + +static double row_dot(const Problem *p, int row, const double *x) { + if (row < p->ms) return x[row]; + const double *a = p->A + (size_t)(row - p->ms) * p->n; + double value = 0.0; + for (int j = 0; j < p->n; ++j) value += a[j] * x[j]; + return value; +} + +/* Scaled KKT residuals following the supplied Maros-Meszaros native runner. */ +static void kkt_residuals(const Problem *p, const double *x, const double *lam, + double *primal, double *dual, double *gap) { + double residual = 0.0, scale = 0.0; + for (int i = 0; i < p->m; ++i) { + const double value = row_dot(p, i, x); + double violation = 0.0; + if (p->bupper[i] < 1e29 && value > p->bupper[i]) + violation = value - p->bupper[i]; + if (p->blower[i] > -1e29 && p->blower[i] - value > violation) + violation = p->blower[i] - value; + if (violation > residual) residual = violation; + if (fabs(value) > scale) scale = fabs(value); + } + *primal = residual / (1.0 + scale); + + double *gradient = calloc((size_t)p->n, sizeof(double)); + double dual_scale = 0.0; + for (int i = 0; i < p->n; ++i) { + double hx = 0.0; + for (int j = 0; j < p->n; ++j) + hx += p->H[(size_t)i * p->n + j] * x[j]; + gradient[i] = hx + p->f[i]; + if (fabs(hx) > dual_scale) dual_scale = fabs(hx); + if (fabs(p->f[i]) > dual_scale) dual_scale = fabs(p->f[i]); + } + for (int i = 0; i < p->m; ++i) { + if (lam[i] == 0.0) continue; + if (fabs(lam[i]) > dual_scale) dual_scale = fabs(lam[i]); + if (i < p->ms) { + gradient[i] += lam[i]; + } else { + const double *a = p->A + (size_t)(i - p->ms) * p->n; + for (int j = 0; j < p->n; ++j) gradient[j] += a[j] * lam[i]; + } + } + residual = 0.0; + for (int i = 0; i < p->n; ++i) + if (fabs(gradient[i]) > residual) residual = fabs(gradient[i]); + *dual = residual / (1.0 + dual_scale); + free(gradient); + + double gap_value = 0.0, gap_scale = 0.0; + for (int i = 0; i < p->m; ++i) { + if (lam[i] == 0.0) continue; + const double bound = lam[i] > 0.0 ? p->bupper[i] : p->blower[i]; + if (fabs(bound) > 1e29) continue; + const double term = lam[i] * (row_dot(p, i, x) - bound); + gap_value += term; + gap_scale += fabs(term); + } + *gap = fabs(gap_value) / (1.0 + gap_scale); +} + +static Result solve_problem(Problem *p, const Tolerance *tolerance, int repeats) { + Result output; + memset(&output, 0, sizeof(output)); + output.solve_time = HUGE_VAL; + double *x = malloc(sizeof(double) * (size_t)p->n); + double *lam = malloc(sizeof(double) * (size_t)p->m); + int *sense = malloc(sizeof(int) * (size_t)p->m); + memcpy(sense, p->sense, sizeof(int) * (size_t)p->m); + + for (int repeat = 0; repeat < repeats; ++repeat) { + DAQPProblem qp = { + p->n, p->m, p->ms, p->H, p->f, p->A, p->bupper, p->blower, + p->sense, NULL, 0, 0 + }; + DAQPSettings settings; + daqp_default_settings(&settings); + if (!tolerance->use_defaults) { + settings.primal_tol = tolerance->solver_tol; + settings.dual_tol = tolerance->solver_tol; + } + settings.time_limit = 1000.0; + + DAQPResult result; + memset(&result, 0, sizeof(result)); + result.x = x; + result.lam = lam; + daqp_quadprog(&result, &qp, &settings); + memcpy(p->sense, sense, sizeof(int) * (size_t)p->m); + + output.exitflag = result.exitflag; + output.iterations = result.iter; + if (result.solve_time < output.solve_time) output.solve_time = result.solve_time; + } + + if (output.exitflag > 0) { + kkt_residuals(p, x, lam, &output.primal, &output.dual, &output.gap); + output.solved = output.primal <= tolerance->check_tol && + output.dual <= tolerance->check_tol && + output.gap <= tolerance->check_tol; + } else { + output.primal = output.dual = output.gap = HUGE_VAL; + } + + free(x); + free(lam); + free(sense); + return output; +} + +int main(int argc, char **argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + const char *directory = argv[1]; + const int repeats = atoi(argv[3]); + if (repeats < 1) return 2; + + char index_path[4096]; + snprintf(index_path, sizeof(index_path), "%s/index.txt", directory); + FILE *index = fopen(index_path, "r"); + FILE *csv = fopen(argv[2], "w"); + if (index == NULL || csv == NULL) { + fprintf(stderr, "could not open benchmark input or output\n"); + return 1; + } + + const Tolerance tolerances[] = { + {"default", 0.0, 1.0, 1}, + {"low", 1e-3, 1e-3, 0}, + {"med", 1e-6, 1e-6, 0}, + {"high", 1e-9, 1e-9, 0}, + }; + fprintf(csv, "problem,n,m,posdef,tolerance,solved,solve_time_s,exitflag," + "iterations,primal_residual,dual_residual,duality_gap\n"); + + char name[256], path[4096]; + int n, m, posdef, problem_count = 0; + while (fscanf(index, "%255s %d %d %d", name, &n, &m, &posdef) == 4) { + snprintf(path, sizeof(path), "%s/%s.bin", directory, name); + Problem problem; + memset(&problem, 0, sizeof(problem)); + if (!read_problem(path, &problem)) { + fprintf(stderr, "could not read %s\n", path); + return 1; + } + ++problem_count; + fprintf(stderr, "[%d] %s (n=%d, m=%d)\n", problem_count, name, n, m); + for (size_t i = 0; i < sizeof(tolerances) / sizeof(tolerances[0]); ++i) { + const Result result = solve_problem(&problem, &tolerances[i], repeats); + fprintf(csv, "%s,%d,%d,%d,%s,%d,%.17g,%d,%d,%.17g,%.17g,%.17g\n", + name, n, m, posdef, tolerances[i].name, result.solved, + result.solve_time, result.exitflag, result.iterations, + result.primal, result.dual, result.gap); + fflush(csv); + } + free_problem(&problem); + } + + fclose(index); + fclose(csv); + return problem_count == 62 ? 0 : 1; +} diff --git a/.github/workflows/ci_benchmark.yml b/.github/workflows/ci_benchmark.yml index 4a1a5e2..e05d21f 100644 --- a/.github/workflows/ci_benchmark.yml +++ b/.github/workflows/ci_benchmark.yml @@ -11,35 +11,80 @@ on: description: "Pull request number to benchmark (leave empty to benchmark the selected branch against master)" required: false default: "" + type: string suite: - description: "Benchmark suite: small, medium, large or all" - required: false + description: "Synthetic benchmark size" + required: true default: "medium" + type: choice + options: + - small + - medium + - large + - all + qp: + description: "Quadratic programs" + required: true + default: true + type: boolean + lp: + description: "Linear programs" + required: true + default: true + type: boolean + miqp: + description: "Mixed-integer quadratic programs" + required: true + default: true + type: boolean + avi: + description: "Affine variational inequalities" + required: true + default: true + type: boolean + equality: + description: "Equality-constrained quadratic programs" + required: true + default: true + type: boolean + maros_meszaros: + description: "Maros–Mészáros dense and positive-definite subsets" + required: true + default: false + type: boolean threshold: description: "Wall time regression threshold in percent (noisy measure)" - required: false - default: "25" + required: true + default: 25 + type: number work_threshold: description: "Iteration/node regression threshold in percent (exact measure)" - required: false - default: "5" + required: true + default: 5 + type: number concurrency: - group: benchmark-${{ github.event.inputs.pr || github.ref }} + group: benchmark-${{ inputs.pr || github.ref }} cancel-in-progress: true jobs: benchmark: - name: Regression check vs base + name: Synthetic regression check vs base + if: inputs.qp || inputs.lp || inputs.miqp || inputs.avi || inputs.equality runs-on: ubuntu-latest permissions: contents: read pull-requests: write env: - SUITE: ${{ github.event.inputs.suite }} - THRESHOLD: ${{ github.event.inputs.threshold }} - WORK_THRESHOLD: ${{ github.event.inputs.work_threshold }} - PR: ${{ github.event.inputs.pr }} + SUITE: ${{ inputs.suite }} + THRESHOLD: ${{ inputs.threshold }} + WORK_THRESHOLD: ${{ inputs.work_threshold }} + PR: ${{ inputs.pr }} + RUN_QP: ${{ inputs.qp }} + RUN_LP: ${{ inputs.lp }} + RUN_MIQP: ${{ inputs.miqp }} + RUN_AVI: ${{ inputs.avi }} + RUN_EQ: ${{ inputs.equality }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -83,6 +128,16 @@ jobs: - name: "Instantiate Julia project" run: julia --color=yes --project=build/interfaces/daqp-julia -e "using Pkg; Pkg.instantiate()" + - name: "Select synthetic problem types" + run: | + problem_types=() + [ "$RUN_QP" = "true" ] && problem_types+=(qp) + [ "$RUN_LP" = "true" ] && problem_types+=(lp) + [ "$RUN_MIQP" = "true" ] && problem_types+=(miqp) + [ "$RUN_AVI" = "true" ] && problem_types+=(avi) + [ "$RUN_EQ" = "true" ] && problem_types+=(eq) + (IFS=,; echo "PROBLEM_TYPES=${problem_types[*]}") >> "$GITHUB_ENV" + - name: "Benchmark against base" id: bench continue-on-error: true @@ -96,7 +151,8 @@ jobs: "$THRESHOLD" \ "$GITHUB_WORKSPACE/benchmark_results" \ 1 \ - "$WORK_THRESHOLD" + "$WORK_THRESHOLD" \ + "$PROBLEM_TYPES" - name: "Write job summary" if: always() @@ -107,7 +163,7 @@ jobs: { echo "## DAQP benchmark: \`${HEAD_SHA:0:12}\` vs \`${BASE_SHA:0:12}\`" echo "" - echo "Suite: \`$SUITE\` · time threshold: \`$THRESHOLD%\` · work threshold: \`$WORK_THRESHOLD%\`" + echo "Suite: \`$SUITE\` · problem types: \`$PROBLEM_TYPES\` · time threshold: \`$THRESHOLD%\` · work threshold: \`$WORK_THRESHOLD%\`" echo "" if [ -f benchmark_results/comparison.txt ]; then echo '```' @@ -131,7 +187,7 @@ jobs: if-no-files-found: warn - name: "Comment results on the pull request" - if: always() && github.event.inputs.pr != '' + if: always() && inputs.pr != '' continue-on-error: true env: BASE_SHA: ${{ steps.refs.outputs.base_sha }} @@ -144,7 +200,7 @@ jobs: echo "### ⚠️ DAQP benchmark: possible regression" fi echo "" - echo "Compared against base \`${BASE_SHA:0:12}\` · suite \`$SUITE\` · time threshold \`$THRESHOLD%\` · work threshold \`$WORK_THRESHOLD%\`" + echo "Compared against base \`${BASE_SHA:0:12}\` · suite \`$SUITE\` · problem types \`$PROBLEM_TYPES\` · time threshold \`$THRESHOLD%\` · work threshold \`$WORK_THRESHOLD%\`" echo "" echo "
Full comparison" echo "" @@ -167,3 +223,129 @@ jobs: run: | echo "Performance regression detected -- see the job summary for details." exit 1 + + maros-meszaros: + name: Maros–Mészáros regression check vs base + if: inputs.maros_meszaros + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + env: + THRESHOLD: ${{ inputs.threshold }} + PR: ${{ inputs.pr }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Resolve commits to compare" + id: refs + run: | + set -euo pipefail + if [ -n "$PR" ]; then + base_ref=$(gh pr view "$PR" --json baseRefName -q .baseRefName) + git fetch --no-tags origin "pull/$PR/head" + git checkout --detach FETCH_HEAD + base_sha=$(git merge-base HEAD "origin/$base_ref") + else + base_sha=$(git merge-base HEAD origin/master) + fi + head_sha=$(git rev-parse HEAD) + if [ "$head_sha" = "$base_sha" ]; then + echo "::error::Head and base are the same commit ($head_sha) -- nothing to compare." + exit 1 + fi + echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT" + echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" + + - name: "Build current DAQP" + run: | + cmake -S . -B build -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DJULIA=ON + cmake --build build --target all --config Release -- -j4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: "Install benchmark data dependencies" + run: python -m pip install "numpy<3" "scipy<2" "qpbenchmark==2.5.0" + + - name: "Check out Maros–Mészáros problems" + uses: actions/checkout@v4 + with: + repository: qpsolvers/maros_meszaros_qpbenchmark + ref: 0cae37f473c2fd0a762e537a61bd4db5b31ff00c + path: maros_meszaros_qpbenchmark + + - name: "Benchmark dense subsets against base" + id: bench + continue-on-error: true + env: + BASE_SHA: ${{ steps.refs.outputs.base_sha }} + run: | + bash .github/benchmarks/maros_meszaros_comparison_git.sh \ + build \ + "$BASE_SHA" \ + maros_meszaros_qpbenchmark/data \ + "$GITHUB_WORKSPACE/maros_meszaros_results" \ + "$THRESHOLD" \ + 3 + + - name: "Write job summary" + if: always() + env: + HEAD_SHA: ${{ steps.refs.outputs.head_sha }} + BASE_SHA: ${{ steps.refs.outputs.base_sha }} + run: | + { + echo "## DAQP Maros–Mészáros benchmark: \`${HEAD_SHA:0:12}\` vs \`${BASE_SHA:0:12}\`" + echo "" + if [ -f maros_meszaros_results/comparison.md ]; then + cat maros_meszaros_results/comparison.md + else + echo "The benchmark did not produce a comparison -- see the job log." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: "Upload Maros–Mészáros results" + if: always() + uses: actions/upload-artifact@v4 + with: + name: maros-meszaros-results + path: | + maros_meszaros_results/*.csv + maros_meszaros_results/comparison.md + if-no-files-found: warn + + - name: "Comment results on the pull request" + if: always() && inputs.pr != '' + continue-on-error: true + env: + BASE_SHA: ${{ steps.refs.outputs.base_sha }} + run: | + { + echo "" + if [ "${{ steps.bench.outcome }}" = "success" ]; then + echo "### ✅ DAQP Maros–Mészáros benchmark: no regression detected" + else + echo "### ⚠️ DAQP Maros–Mészáros benchmark: possible regression" + fi + echo "" + echo "Compared against base \`${BASE_SHA:0:12}\` · time threshold \`$THRESHOLD%\`" + echo "" + if [ -f maros_meszaros_results/comparison.md ]; then + cat maros_meszaros_results/comparison.md + else + echo "No comparison produced -- see the job log." + fi + } > maros_comment.md + gh pr comment "$PR" --body-file maros_comment.md --edit-last --create-if-none + + - name: "Fail if a regression was detected" + if: steps.bench.outcome == 'failure' + run: | + echo "Performance regression detected -- see the job summary for details." + exit 1 diff --git a/interfaces/daqp-julia/test/benchmark.jl b/interfaces/daqp-julia/test/benchmark.jl index 2654653..31d3ba7 100644 --- a/interfaces/daqp-julia/test/benchmark.jl +++ b/interfaces/daqp-julia/test/benchmark.jl @@ -14,6 +14,7 @@ Usage: julia benchmark.jl # Run benchmarks with default settings julia benchmark.jl --output results.csv # Specify output file julia benchmark.jl --suite small # Run only small problems + julia benchmark.jl --problem-types qp,avi # Select benchmark families julia benchmark.jl --prox # Run semi-proximal vs full-proximal comparison """ @@ -462,10 +463,24 @@ function print_stats(stats; show_nodes=false) println(line) end -function run_benchmarks(; suite="all", output_file="daqp_benchmark_results.csv", use_local=false) +const BENCHMARK_TYPES = ["qp", "lp", "eq", "miqp", "avi"] + +function parse_problem_types(value) + selected = Set(lowercase.(strip.(split(value, ",")))) + unknown = setdiff(selected, Set(BENCHMARK_TYPES)) + isempty(unknown) || + error("Unknown problem type(s): $(join(sort(collect(unknown)), ", ")). " * + "Must be a comma-separated subset of: $(join(BENCHMARK_TYPES, ", "))") + isempty(selected) && error("At least one problem type must be selected") + return selected +end + +function run_benchmarks(; suite="all", output_file="daqp_benchmark_results.csv", + use_local=false, problem_types=join(BENCHMARK_TYPES, ",")) """ Run all benchmarks and save results to CSV. suite: "small", "medium", "large", or "all" + problem_types: comma-separated subset of "qp", "lp", "eq", "miqp", "avi" """ # Use local libdaqp if available @@ -498,68 +513,80 @@ function run_benchmarks(; suite="all", output_file="daqp_benchmark_results.csv", timestamp = string(now()) version = string(pkgversion(DAQPBase)) + selected_types = parse_problem_types(problem_types) @info "Starting DAQP performance benchmarks..." @info "Running suite: $suite" + @info "Problem types: $(join(sort(collect(selected_types)), ", "))" # Run QP benchmarks - println("\n=== Quadratic Programming Benchmarks ===") - for (i, (n, m, ms, nAct, kappa)) in enumerate(sizes_to_run) - problem_id = "qp_$(n)_$(m)_$(ms)_$(nAct)_$(Int(log10(kappa)))" - println(" [$i/$(length(sizes_to_run))] QP: n=$n, m=$m, ms=$ms, nAct=$nAct, κ=$kappa") - - stats = benchmark_qp(n, m, ms, nAct, kappa) - push!(csv_lines, benchmark_csv_row(timestamp, version, "QP", problem_id, - n, m, ms, kappa, stats)) - print_stats(stats) + if "qp" in selected_types + println("\n=== Quadratic Programming Benchmarks ===") + for (i, (n, m, ms, nAct, kappa)) in enumerate(sizes_to_run) + problem_id = "qp_$(n)_$(m)_$(ms)_$(nAct)_$(Int(log10(kappa)))" + println(" [$i/$(length(sizes_to_run))] QP: n=$n, m=$m, ms=$ms, nAct=$nAct, κ=$kappa") + + stats = benchmark_qp(n, m, ms, nAct, kappa) + push!(csv_lines, benchmark_csv_row(timestamp, version, "QP", problem_id, + n, m, ms, kappa, stats)) + print_stats(stats) + end end # Run LP benchmarks - println("\n=== Linear Programming Benchmarks ===") - for (i, (n, m, ms, _, _)) in enumerate(sizes_to_run) - problem_id = "lp_$(n)_$(m)_$(ms)" - println(" [$i/$(length(sizes_to_run))] LP: n=$n, m=$m, ms=$ms") - - stats = benchmark_lp(n, m, ms) - push!(csv_lines, benchmark_csv_row(timestamp, version, "LP", problem_id, - n, m, ms, "", stats)) - print_stats(stats) + if "lp" in selected_types + println("\n=== Linear Programming Benchmarks ===") + for (i, (n, m, ms, _, _)) in enumerate(sizes_to_run) + problem_id = "lp_$(n)_$(m)_$(ms)" + println(" [$i/$(length(sizes_to_run))] LP: n=$n, m=$m, ms=$ms") + + stats = benchmark_lp(n, m, ms) + push!(csv_lines, benchmark_csv_row(timestamp, version, "LP", problem_id, + n, m, ms, "", stats)) + print_stats(stats) + end end # Run equality constrained QP benchmarks - println("\n=== Equality Constrained QP Benchmarks ===") - for (i, (n, m, ms, nAct, neq, kappa)) in enumerate(eq_sizes_to_run) - problem_id = "eqqp_$(n)_$(m)_$(ms)_$(nAct)_$(neq)" - println(" [$i/$(length(eq_sizes_to_run))] EQ-QP: n=$n, m=$m, ms=$ms, nAct=$nAct, neq=$neq") - - stats = benchmark_eq_qp(n, m, ms, nAct, neq, kappa) - push!(csv_lines, benchmark_csv_row(timestamp, version, "EQ-QP", problem_id, - n, m, ms, kappa, stats)) - print_stats(stats) + if "eq" in selected_types + println("\n=== Equality Constrained QP Benchmarks ===") + for (i, (n, m, ms, nAct, neq, kappa)) in enumerate(eq_sizes_to_run) + problem_id = "eqqp_$(n)_$(m)_$(ms)_$(nAct)_$(neq)" + println(" [$i/$(length(eq_sizes_to_run))] EQ-QP: n=$n, m=$m, ms=$ms, nAct=$nAct, neq=$neq") + + stats = benchmark_eq_qp(n, m, ms, nAct, neq, kappa) + push!(csv_lines, benchmark_csv_row(timestamp, version, "EQ-QP", problem_id, + n, m, ms, kappa, stats)) + print_stats(stats) + end end # Run mixed-integer QP benchmarks - println("\n=== Mixed-Integer QP (branch and bound) Benchmarks ===") - for (i, (n, m, ms, nb)) in enumerate(miqp_sizes_to_run) - problem_id = "miqp_$(n)_$(m)_$(ms)_$(nb)" - println(" [$i/$(length(miqp_sizes_to_run))] MIQP: n=$n, m=$m, ms=$ms, nb=$nb") - - stats = benchmark_miqp(n, m, ms, nb) - push!(csv_lines, benchmark_csv_row(timestamp, version, "MIQP", problem_id, - n, m, ms, "", stats)) - print_stats(stats; show_nodes=true) + if "miqp" in selected_types + println("\n=== Mixed-Integer QP (branch and bound) Benchmarks ===") + for (i, (n, m, ms, nb)) in enumerate(miqp_sizes_to_run) + problem_id = "miqp_$(n)_$(m)_$(ms)_$(nb)" + println(" [$i/$(length(miqp_sizes_to_run))] MIQP: n=$n, m=$m, ms=$ms, nb=$nb") + + stats = benchmark_miqp(n, m, ms, nb) + push!(csv_lines, benchmark_csv_row(timestamp, version, "MIQP", problem_id, + n, m, ms, "", stats)) + print_stats(stats; show_nodes=true) + end end # Run AVI benchmarks - println("\n=== Affine Variational Inequality Benchmarks ===") - for (i, (n, m)) in enumerate(avi_sizes_to_run) - problem_id = "avi_$(n)_$(m)" - println(" [$i/$(length(avi_sizes_to_run))] AVI: n=$n, m=$m") - - stats = benchmark_avi(n, m) - push!(csv_lines, benchmark_csv_row(timestamp, version, "AVI", problem_id, - n, m, 0, "", stats)) - print_stats(stats) + if "avi" in selected_types + println("\n=== Affine Variational Inequality Benchmarks ===") + for (i, (n, m)) in enumerate(avi_sizes_to_run) + problem_id = "avi_$(n)_$(m)" + println(" [$i/$(length(avi_sizes_to_run))] AVI: n=$n, m=$m") + + stats = benchmark_avi(n, m) + push!(csv_lines, benchmark_csv_row(timestamp, version, "AVI", problem_id, + n, m, 0, "", stats)) + print_stats(stats) + end end # Save to CSV @@ -578,6 +605,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ local output_file = "daqp_benchmark_results.csv" local use_local = false local run_prox = false + local problem_types = join(BENCHMARK_TYPES, ",") local i = 1 while i <= length(ARGS) if ARGS[i] == "--output" && i < length(ARGS) @@ -586,6 +614,9 @@ if abspath(PROGRAM_FILE) == @__FILE__ elseif ARGS[i] == "--suite" && i < length(ARGS) suite = ARGS[i+1] i += 2 + elseif ARGS[i] == "--problem-types" && i < length(ARGS) + problem_types = ARGS[i+1] + i += 2 elseif ARGS[i] == "--local" use_local = true i += 1 @@ -600,6 +631,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ if run_prox run_prox_benchmark(; output_file="prox_comparison.csv", use_local=use_local) else - run_benchmarks(; suite=suite, output_file=output_file, use_local=use_local) + run_benchmarks(; suite=suite, output_file=output_file, use_local=use_local, + problem_types=problem_types) end end diff --git a/interfaces/daqp-julia/test/benchmark_comparison_git.sh b/interfaces/daqp-julia/test/benchmark_comparison_git.sh index ecad899..fa14cde 100755 --- a/interfaces/daqp-julia/test/benchmark_comparison_git.sh +++ b/interfaces/daqp-julia/test/benchmark_comparison_git.sh @@ -9,7 +9,7 @@ # exact same problems and the CSV problem_ids line up. # # Usage: -# benchmark_comparison_git.sh [ref] [suite] [threshold] [outdir] [fail_on_regression] [work_threshold] +# benchmark_comparison_git.sh [ref] [suite] [threshold] [outdir] [fail_on_regression] [work_threshold] [problem_types] set -e @@ -20,6 +20,7 @@ REGRESSION_THRESHOLD="${4:-5}" OUTPUT_DIR="${5:-.}" FAIL_ON_REGRESSION="${6:-1}" WORK_THRESHOLD="${7:-5}" +PROBLEM_TYPES="${8:-qp,lp,eq,miqp,avi}" # Convert to absolute paths JULIA_PROJECT="$(cd "$JULIA_PROJECT" && pwd)" @@ -39,6 +40,7 @@ echo "==========================================" echo "Julia project: $JULIA_PROJECT" echo "Baseline ref: $GIT_REF" echo "Benchmark suite: $BENCHMARK_SUITE" +echo "Problem types: $PROBLEM_TYPES" echo "Time regression threshold: $REGRESSION_THRESHOLD%" echo "Work regression threshold: $WORK_THRESHOLD%" echo "Output directory: $OUTPUT_DIR" @@ -59,6 +61,7 @@ fi julia --project="$JULIA_PROJECT" "$BENCHMARK_SCRIPT" \ --suite "$BENCHMARK_SUITE" \ + --problem-types "$PROBLEM_TYPES" \ --output "$OUTPUT_DIR/current_dev.csv" \ --local \ 2>&1 | grep -E "(Starting|Mean|Results|Using)" || true @@ -137,6 +140,7 @@ cd "$REPO_ROOT" echo "Step 3: Running benchmarks with baseline $GIT_REF..." julia --project="$JULIA_PROJECT" "$BENCHMARK_SCRIPT" \ --suite "$BENCHMARK_SUITE" \ + --problem-types "$PROBLEM_TYPES" \ --output "$OUTPUT_DIR/old_version.csv" \ --local \ 2>&1 | grep -E "(Starting|Mean|Results|Using)" || true