cmd/mfp-test: Phase 4 - test matrix and batch/quick/single modes

cmd/mfp-test/test/capture.go

@@ -85,3 +85,13 @@ func (dc *DocumentCapture) Docs() []CapturedDoc {
 	copy(out, dc.docs)
 	return out
 }
+
+// Reset clears all captured documents and resets the OnDocument
+// signal so the capture can be reused for the next test run.
+func (dc *DocumentCapture) Reset() {
+	dc.mu.Lock()
+	defer dc.mu.Unlock()
+
+	dc.docs = nil
+	dc.done = make(chan struct{})
+}

cmd/mfp-test/test/command.go

@@ -15,9 +15,7 @@ import (
 	"image/png"
 	"net"
 	"os"
-	"os/exec"
 	"strconv"
-	"time"
 
 	"github.com/OpenPrinting/go-mfp/argv"
 	"github.com/OpenPrinting/go-mfp/log"
@@ -89,11 +87,15 @@ var Command = argv.Command{
 		},
 		{
 			Name:      "--single",
-			Help:      "run a single test configuration by name",
+			Help:      "run a single test configuration by name (sides/color-mode/format)",
 			HelpArg:   "name",
 			Singleton: true,
 			Validate:  argv.ValidateAny,
 		},
+		{
+			Name: "--quick",
+			Help: "run reduced test matrix (all sides × all color modes, first format only)",
+		},
 		{
 			Name:    "-v",
 			Aliases: []string{"--verbose"},
@@ -182,33 +184,66 @@ func cmdTestHandler(ctx context.Context, inv *argv.Invocation) error {
 
 	log.Info(ctx, "CUPS queue %q ready at %s", queueName, ippURL)
 
-	// Generate a test PNG image and send it through the full pipeline
-	imgPath, err := generateTestPNG()
+	// Query printer capabilities for test matrix generation.
+	caps, err := QueryPrinterCaps(ctx, ippURL)
 	if err != nil {
-		return fmt.Errorf("generate test image: %w", err)
+		return fmt.Errorf("query printer capabilities: %w", err)
 	}
-	defer os.Remove(imgPath)
 
-	log.Info(ctx, "sending test PNG via lp...")
-	lpCmd := exec.CommandContext(ctx, "lp", "-d", queueName, imgPath)
-	if out, err := lpCmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("lp -d %s: %w: %s", queueName, err, out)
+	// --list: print all configurations and exit.
+	if inv.Flag("--list") {
+		configs := BatchMatrix(caps)
+		for _, cfg := range configs {
+			fmt.Println(cfg.Name)
+		}
+		return nil
 	}
 
-	// Wait for the document to arrive at the capture backend
-	select {
-	case <-capture.OnDocument():
-	case <-time.After(30 * time.Second):
-		return fmt.Errorf("timeout: no document received after 30s")
-	case <-ctx.Done():
-		return nil
+	// Determine which test configurations to run.
+	var configs []TestConfig
+	switch {
+	case inv.Flag("--batch"):
+		configs = BatchMatrix(caps)
+	case inv.Flag("--quick"):
+		configs = QuickMatrix(caps)
+	default:
+		if spec, ok := inv.Get("--single"); ok {
+			cfg, err := SingleConfig(spec)
+			if err != nil {
+				return err
+			}
+			configs = []TestConfig{*cfg}
+		} else {
+			// Default: single run with printer defaults.
+			configs = QuickMatrix(caps)
+		}
 	}
 
-	// Report what was captured
-	docs := capture.Docs()
-	for i, d := range docs {
-		log.Info(ctx, "captured doc %d: %d bytes, format=%q, job=%q",
-			i+1, len(d.Data), d.Params.Format, d.Params.JobName)
+	// Parse similarity threshold.
+	threshold := DefaultThreshold
+	if ts, ok := inv.Get("--threshold"); ok {
+		var t float64
+		if _, err := fmt.Sscanf(ts, "%f", &t); err != nil {
+			return fmt.Errorf("invalid threshold %q: %w", ts, err)
+		}
+		threshold = t
+	}
+
+	verbose := inv.Flag("-v")
+
+	// Run each test configuration.
+	for _, cfg := range configs {
+		log.Info(ctx, "running test: %s", cfg.Name)
+		result, err := RunTest(ctx, cfg, queueName, capture, threshold, verbose)
+		if err != nil {
+			log.Info(ctx, "FAIL %s: %v", cfg.Name, err)
+			continue
+		}
+		if result.Passed {
+			log.Info(ctx, "PASS %s (score=%.4f)", cfg.Name, result.Score)
+		} else {
+			log.Info(ctx, "FAIL %s (score=%.4f < threshold=%.4f)", cfg.Name, result.Score, threshold)
+		}
 	}
 
 	return nil

cmd/mfp-test/test/matrix.go

@@ -0,0 +1,137 @@
+// MFP - Multi-Function Printers and scanners toolkit
+//
+// Copyright (C) 2026 Mohammad Arman (officialmdarman@gmail.com)
+// See LICENSE for license terms and conditions
+//
+// Test matrix generation for mfp-test
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/OpenPrinting/go-mfp/proto/ipp"
+)
+
+// PrinterCaps holds the queried printer capabilities used to generate
+// the test matrix.
+type PrinterCaps struct {
+	Sides      []ipp.KwSides
+	ColorModes []string
+	Formats    []string
+}
+
+// TestConfig represents one specific combination of print parameters
+// to exercise in a test run.
+type TestConfig struct {
+	Name      string
+	Sides     ipp.KwSides
+	ColorMode string
+	Format    string
+}
+
+// QueryPrinterCaps queries the virtual IPP printer for its supported
+// attribute values and returns them as a PrinterCaps.
+func QueryPrinterCaps(ctx context.Context, printerURL string) (*PrinterCaps, error) {
+	u, err := url.Parse(printerURL)
+	if err != nil {
+		return nil, fmt.Errorf("matrix: parse printer URL: %w", err)
+	}
+
+	client := ipp.NewClient(u, nil)
+	attrs, err := client.GetPrinterAttributes(ctx,
+		[]string{"job-template", "printer-description"}, "")
+	if err != nil {
+		return nil, fmt.Errorf("matrix: query printer attributes: %w", err)
+	}
+
+	caps := &PrinterCaps{
+		Sides:      attrs.SidesSupported,
+		ColorModes: attrs.PrintColorModeSupported,
+		Formats:    attrs.DocumentFormatSupported,
+	}
+
+	if len(caps.Sides) == 0 {
+		caps.Sides = []ipp.KwSides{ipp.KwSidesOneSided}
+	}
+	if len(caps.ColorModes) == 0 {
+		caps.ColorModes = []string{"color"}
+	}
+	if len(caps.Formats) == 0 {
+		caps.Formats = []string{"application/octet-stream"}
+	}
+
+	return caps, nil
+}
+
+// configName builds a deterministic, human-readable name for a test
+// configuration from its three dimensions.
+func configName(sides ipp.KwSides, color, format string) string {
+	return fmt.Sprintf("%s/%s/%s", sides, color, format)
+}
+
+// BatchMatrix returns every combination of sides × color mode × format.
+// This is the exhaustive test matrix.
+func BatchMatrix(caps *PrinterCaps) []TestConfig {
+	var configs []TestConfig
+	for _, sides := range caps.Sides {
+		for _, color := range caps.ColorModes {
+			for _, format := range caps.Formats {
+				configs = append(configs, TestConfig{
+					Name:      configName(sides, color, format),
+					Sides:     sides,
+					ColorMode: color,
+					Format:    format,
+				})
+			}
+		}
+	}
+	return configs
+}
+
+// QuickMatrix returns a reduced matrix: all sides × all color modes,
+// but only the first document format. Duplex/simplex and color/mono
+// are tested independently; format variation is omitted to keep the
+// run short.
+func QuickMatrix(caps *PrinterCaps) []TestConfig {
+	format := caps.Formats[0]
+	var configs []TestConfig
+	for _, sides := range caps.Sides {
+		for _, color := range caps.ColorModes {
+			configs = append(configs, TestConfig{
+				Name:      configName(sides, color, format),
+				Sides:     sides,
+				ColorMode: color,
+				Format:    format,
+			})
+		}
+	}
+	return configs
+}
+
+// SingleConfig parses a configuration name of the form
+// "sides/color-mode/format" and returns the corresponding TestConfig.
+// This is used with --single to reproduce a specific known bug.
+func SingleConfig(spec string) (*TestConfig, error) {
+	parts := strings.SplitN(spec, "/", 3)
+	if len(parts) != 3 {
+		return nil, fmt.Errorf("matrix: --single requires sides/color-mode/format, got %q", spec)
+	}
+	sides := ipp.KwSides(parts[0])
+	switch sides {
+	case ipp.KwSidesOneSided,
+		ipp.KwSidesTwoSidedLongEdge,
+		ipp.KwSidesTwoSidedShortEdge:
+	default:
+		return nil, fmt.Errorf("matrix: unknown sides value %q", sides)
+	}
+	return &TestConfig{
+		Name:      spec,
+		Sides:     sides,
+		ColorMode: parts[1],
+		Format:    parts[2],
+	}, nil
+}

cmd/mfp-test/test/runner.go

@@ -0,0 +1,95 @@
+// MFP - Multi-Function Printers and scanners toolkit
+//
+// Copyright (C) 2026 Mohammad Arman (officialmdarman@gmail.com)
+// See LICENSE for license terms and conditions
+//
+// Test runner for mfp-test
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/OpenPrinting/go-mfp/log"
+)
+
+// DefaultThreshold is the minimum similarity score required to pass a test.
+const DefaultThreshold = 0.95
+
+// TestResult holds the outcome of a single test run.
+type TestResult struct {
+	Config  TestConfig
+	Score   float64
+	Passed  bool
+	Details map[string]float64
+}
+
+// RunTest runs a single print test using the given configuration:
+// generates a test PNG, sends it to the CUPS queue with the specified
+// job attributes, waits for capture, and returns the test result.
+//
+// Image evaluation is not yet implemented; the function currently
+// reports success if the document was captured within the timeout.
+func RunTest(ctx context.Context, cfg TestConfig, queueName string,
+	capture *DocumentCapture, threshold float64, verbose bool) (*TestResult, error) {
+
+	// Reset capture so we get only this job's document.
+	capture.Reset()
+
+	// Generate a fresh test image for this run.
+	imgPath, err := generateTestPNG()
+	if err != nil {
+		return nil, fmt.Errorf("generate test image: %w", err)
+	}
+	defer os.Remove(imgPath)
+
+	// Build lp options for the test configuration.
+	lpArgs := []string{"-d", queueName}
+	if cfg.Sides != "" {
+		lpArgs = append(lpArgs, "-o", "sides="+string(cfg.Sides))
+	}
+	if cfg.ColorMode != "" {
+		lpArgs = append(lpArgs, "-o", "print-color-mode="+cfg.ColorMode)
+	}
+	lpArgs = append(lpArgs, imgPath)
+
+	log.Info(ctx, "lp %v", lpArgs)
+	lpCmd := exec.CommandContext(ctx, "lp", lpArgs...)
+	if out, err := lpCmd.CombinedOutput(); err != nil {
+		return nil, fmt.Errorf("lp: %w: %s", err, out)
+	}
+
+	// Wait for the document to arrive.
+	select {
+	case <-capture.OnDocument():
+	case <-time.After(30 * time.Second):
+		return nil, fmt.Errorf("timeout: no document received after 30s")
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
+	docs := capture.Docs()
+	if len(docs) == 0 {
+		return nil, fmt.Errorf("capture returned no documents")
+	}
+
+	d := docs[len(docs)-1]
+	if verbose {
+		log.Info(ctx, "captured %d bytes format=%q job=%q",
+			len(d.Data), d.Params.Format, d.Params.JobName)
+	}
+
+	// Image evaluation will be wired here in Phase 5 once raster
+	// conversion (captured bytes → PNG) is implemented. For now,
+	// a successful capture counts as a pass with a placeholder score.
+	score := 1.0
+	return &TestResult{
+		Config: cfg,
+		Score:  score,
+		Passed: score >= threshold,
+	}, nil
+}