feat(compliance): report how many files each run examined

.github/actions/ci-compliance/action.yml

@@ -130,19 +130,39 @@ runs:
         # DatasetComplianceRunner has a different CLI shape: no check-type
         # argument, no --org-url, and all positional args are treated as file
         # paths. Branch the invocation accordingly.
+        # Tee'd rather than piped straight through: the coverage line has to reach the job
+        # summary, and the runner's own stdout is inside the group above, which the UI
+        # collapses by default. A count nobody expands is not a count anybody reads.
         if ($check -eq "dataset") {
-          & $runnerExe --output github @fileList
+          $runnerOutput = & $runnerExe --output github @fileList 2>&1
         } else {
-          & $runnerExe $check --output github --org-url "$orgUrl" @fileList
+          $runnerOutput = & $runnerExe $check --output github --org-url "$orgUrl" @fileList 2>&1
         }
         $exitCode = $LASTEXITCODE
+
+        # Re-emitted verbatim. Capturing is what lets the coverage line reach the summary below;
+        # the annotations and workflow commands in this output still have to reach the log, and
+        # Write-Host is what puts them there.
+        $runnerOutput | ForEach-Object { Write-Host $_ }
         Write-Host "::endgroup::"
 
         if ($env:GITHUB_STEP_SUMMARY) {
           $status = if ($exitCode -eq 0) { "passed" } else { "**failed**" }
-          @(
-            "### $check compliance $status"
-          ) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
+
+          # One well-defined line the runner controls. Absent rather than fatal if the runner
+          # is older than this action, since the two version independently.
+          $coverage = $runnerOutput |
+                      Where-Object { $_ -match '^Coverage: ' } |
+                      Select-Object -First 1
+
+          $summary = @("### $check compliance $status")
+          if ($coverage) {
+            # Verbatim. Rewording it here would put the same sentence in two places and let
+            # them drift; the runner owns the wording and this owns where it appears.
+            $summary += ""
+            $summary += $coverage
+          }
+          $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
         }
 
         exit $exitCode

.github/scripts/tests/ci-compliance-action.Tests.ps1

@@ -0,0 +1,43 @@
+# ci-compliance-action.Tests.ps1 — structural assertions over ci-compliance's action.yml.
+#
+# The same shape as the sibling file for ci-serialisation: properties of the action as a whole
+# that a reader cannot check from any one place in it.
+#
+# The property here is where the runner's output ends up. The invocation is wrapped in a
+# ::group::, which the GitHub UI collapses by default, so anything the runner prints is hidden
+# until someone expands it. The job summary is the surface a reader actually sees, and today it
+# carries a single line saying only whether the check passed. A run that examined every file and
+# a run that examined none produce the same summary.
+#
+# Run locally:  pwsh -Command "Invoke-Pester .github/scripts/tests -Output Detailed"
+# Run in CI:    lint-workflows.yml, the powershell-tests job.
+
+BeforeAll {
+    $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../../..')).Path
+    $script:actionPath = Join-Path $repoRoot '.github/actions/ci-compliance/action.yml'
+    $script:lines = Get-Content $actionPath
+    $script:text  = $lines -join "`n"
+}
+
+Describe 'ci-compliance action.yml' {
+
+    Context 'the job summary says how much was examined' {
+
+        # Stdout is not the surface. The runner's output is inside a collapsed group, so a
+        # count printed there is invisible to a reader who does not already suspect something.
+        It 'wraps the runner invocation in a collapsed group' {
+            $text | Should -Match '::group::' -Because 'if this stops being true the reasoning below needs revisiting'
+        }
+
+        It 'carries the examined count into the summary, not only pass or fail' {
+            $text | Should -Match '\$summary \+= \$coverage' -Because 'a run that examined every file and one that examined none must not produce the same summary'
+        }
+
+        # The count is the runner's, read back off one line it controls. If that line is ever
+        # renamed on one side only, the summary silently loses the count rather than breaking,
+        # so both ends of the contract are asserted here.
+        It 'reads the count off the line the runner emits' {
+            $text | Should -Match "match '\^Coverage: '"
+        }
+    }
+}

tools/ComplianceRunner/src/ComplianceRunner/ComplianceRunner.cs

@@ -38,16 +38,18 @@ static int Main(string[] args)
 
         var mergedResult   = new TestResult() { Status = TestStatus.Pass, Information = new List<ITestInformation>() };
         var allAnnotations = new List<Annotation>();
+        var accounting     = new FileAccounting(files.Count);
 
         foreach (var file in files)
         {
             // Each check type is only relevant to certain file extensions.
-            if (!FileFilter.IsRelevantFile(file, checkType)) continue;
+            if (!FileFilter.IsRelevantFile(file, checkType)) { accounting.CountNotRelevant(); continue; }
 
             if (verbose) Console.WriteLine($"\n=== Checking: {file} ===");
 
             if (!File.Exists(file))
             {
+                accounting.CountNotOnDisk();
                 Console.WriteLine($"  [SKIP] File not found: {file}");
                 continue;
             }
@@ -114,12 +116,14 @@ static int Main(string[] args)
 
             if (resultForThisFile == null)
             {
+                accounting.CountNoResult();
                 Console.WriteLine($"  [SKIP] No result returned for: {file}");
                 continue;
             }
 
             if (verbose) Console.WriteLine($"  Result Status: {resultForThisFile.Status}");
 
+            accounting.CountExamined();
             mergedResult = mergedResult.Merge(resultForThisFile);
 
             // Code/copyright/documentation findings have been through GroupErrors, whose
@@ -156,7 +160,8 @@ static int Main(string[] args)
         }
 
         OutputEmitter.Write(outputFormat, checkType, mergedResult.Status, allAnnotations, sarifFilePath, verbose,
-            BH.Engine.Base.Query.DocumentationURL("DevOps/Code%20Compliance%20and%20CI/Compliance%20Checks/"));
+            BH.Engine.Base.Query.DocumentationURL("DevOps/Code%20Compliance%20and%20CI/Compliance%20Checks/"),
+            accounting);
 
         // Exit code mirrors BHoMBot: failure only on Error; Warning and Pass are both success.
         return mergedResult.Status == TestStatus.Error ? 1 : 0;

tools/ComplianceRunner/src/DatasetComplianceRunner/DatasetComplianceRunner.cs

@@ -37,16 +37,18 @@ static int Main(string[] args)
 
         var mergedResult   = new TestResult() { Status = TestStatus.Pass, Information = new List<ITestInformation>() };
         var allAnnotations = new List<Annotation>();
+        var accounting     = new FileAccounting(files.Count);
 
         foreach (var file in files)
         {
             // Only .json files under a datasets/ path are in scope.
-            if (!FileFilter.IsDatasetFile(file)) continue;
+            if (!FileFilter.IsDatasetFile(file)) { accounting.CountNotRelevant(); continue; }
 
             if (verbose) Console.WriteLine($"\n=== Checking: {file} ===");
 
             if (!File.Exists(file))
             {
+                accounting.CountNotOnDisk();
                 Console.WriteLine($"  [SKIP] File not found: {file}");
                 continue;
             }
@@ -55,12 +57,14 @@ static int Main(string[] args)
 
             if (resultForThisFile == null)
             {
+                accounting.CountNoResult();
                 Console.WriteLine($"  [SKIP] No result returned for: {file}");
                 continue;
             }
 
             if (verbose) Console.WriteLine($"  Result Status: {resultForThisFile.Status}");
 
+            accounting.CountExamined();
             mergedResult = mergedResult.Merge(resultForThisFile);
 
             var information        = resultForThisFile.Information ?? Enumerable.Empty<ITestInformation>();
@@ -94,7 +98,8 @@ static int Main(string[] args)
 
         const string checkType = "dataset";
         OutputEmitter.Write(outputFormat, checkType, mergedResult.Status, allAnnotations, sarifFilePath, verbose,
-            BH.Engine.Base.Query.DocumentationURL("DevOps/Code%20Compliance%20and%20CI/Compliance%20Checks/"));
+            BH.Engine.Base.Query.DocumentationURL("DevOps/Code%20Compliance%20and%20CI/Compliance%20Checks/"),
+            accounting);
 
         // Exit code mirrors BHoMBot: failure only on Error; Warning and Pass are both success.
         return mergedResult.Status == TestStatus.Error ? 1 : 0;

tools/ComplianceRunner/src/Shared/Compliance.Shared/FileAccounting.cs

@@ -0,0 +1,90 @@
+using System.Collections.Generic;
+
+/// <summary>
+/// Counts what a compliance runner did with the files it was handed, and renders that as a
+/// coverage denominator.
+///
+/// Both runners walk the same four-exit loop: a file is dropped by the filter, dropped because
+/// it is not on disk, dropped because the engine returned no result, or examined. Without a
+/// count of those, a run that examined every file and a run that examined none produce the same
+/// output, so a green check carries no evidence that anything was checked.
+///
+/// Deliberately free of BHoM types and of any dependency on either runner, so it can be
+/// compiled directly into the hermetic test project and its arithmetic and wording tested
+/// without a BHoM install. It holds counters and formats strings; it decides nothing.
+///
+/// The drop reasons are counted but not sub-classified. A file dropped by the filter may have
+/// been excluded on purpose — project compliance deliberately skips test projects and anything
+/// under .ci/ — or may be a disagreement between the pathspec that selected it and the filter
+/// that rejected it. Telling those apart would require the filter to explain itself, which is a
+/// change to a class both runners share. The denominator's job is to make the ratio visible;
+/// diagnosing it is a separate question.
+/// </summary>
+public sealed class FileAccounting
+{
+    /// <summary>Files handed to the runner on the command line.</summary>
+    public int HandedIn { get; private set; }
+
+    /// <summary>Dropped because the check's own filter did not consider them relevant.</summary>
+    public int NotRelevant { get; private set; }
+
+    /// <summary>Dropped because the path did not exist on disk.</summary>
+    public int NotOnDisk { get; private set; }
+
+    /// <summary>Dropped because the compliance engine returned no result for them.</summary>
+    public int NoResult { get; private set; }
+
+    /// <summary>Files the runner actually examined and merged into its verdict.</summary>
+    public int Examined { get; private set; }
+
+    public FileAccounting(int handedIn) => HandedIn = handedIn;
+
+    public void CountNotRelevant() => NotRelevant++;
+    public void CountNotOnDisk()   => NotOnDisk++;
+    public void CountNoResult()    => NoResult++;
+    public void CountExamined()    => Examined++;
+
+    /// <summary>
+    /// True when files were supplied and none of them was examined. The calling workflow exits
+    /// before invoking a runner when it has no files at all, so this is the anomalous case
+    /// rather than the empty one: something selected these files and the runner used none.
+    /// </summary>
+    public bool ExaminedNothing => HandedIn > 0 && Examined == 0;
+
+    /// <summary>
+    /// The coverage line. Leads with what was examined rather than with what was handed in:
+    /// the whole point is to make a green interpretable, and a count of inputs does not.
+    /// </summary>
+    public string CoverageLine() =>
+        $"Coverage: {Examined} of {HandedIn} file(s) examined; {DropSummary()}.";
+
+    /// <summary>
+    /// The warning text for a run that examined nothing. Carries the breakdown rather than the
+    /// bare fact, because the breakdown is what distinguishes a pull request that changed
+    /// nothing relevant from a selection layer handing over files this check will never accept.
+    /// </summary>
+    public string ExaminedNothingWarning() =>
+        $"{HandedIn} file(s) were handed to this check and none was examined: {DropSummary()}. "
+      + "The check reports success without having inspected anything. If the files were dropped "
+      + "as not relevant, the pattern that selected them and the filter that rejected them "
+      + "disagree.";
+
+    private string DropSummary()
+    {
+        var parts = new List<string>();
+        if (NotRelevant > 0) parts.Add($"{NotRelevant} not relevant to this check");
+        if (NotOnDisk   > 0) parts.Add($"{NotOnDisk} not found on disk");
+        if (NoResult    > 0) parts.Add($"{NoResult} returned no result");
+        return parts.Count == 0 ? "none dropped" : string.Join(", ", parts);
+    }
+
+    /// <summary>Shape for machine-readable output, so consumers get counts structurally.</summary>
+    public Dictionary<string, object> ToPayload() => new()
+    {
+        ["handedIn"]    = HandedIn,
+        ["examined"]    = Examined,
+        ["notRelevant"] = NotRelevant,
+        ["notOnDisk"]   = NotOnDisk,
+        ["noResult"]    = NoResult,
+    };
+}

tools/ComplianceRunner/src/Shared/Compliance.Shared/OutputEmitter.cs

@@ -26,10 +26,32 @@ public static void Write(
         List<Annotation> annotations,
         string? sarifFilePath,
         bool verbose,
-        string toolUri)
+        string toolUri,
+        FileAccounting? accounting = null)
     {
         CheckMetadata.GetOutput(checkType, status, out string title, out string summary, out string text);
 
+        // Coverage goes to stdout only for the two human-facing formats. json and sarif put a
+        // payload on stdout and nothing else may precede it: an unconditional Console.WriteLine
+        // here is exactly how the [SKIP] diagnostic already breaks a caller that parses stdout
+        // directly. The counts still reach machine-readable consumers, structurally, below.
+        if (accounting is not null && (outputFormat == "console" || outputFormat == "github"))
+        {
+            Console.WriteLine(accounting.CoverageLine());
+
+            // Reported, never failed. Whether examining nothing should fail is a separate and
+            // undecided question; this only makes the state visible, and it is emitted as a
+            // workflow command so it survives the collapsed log group the caller wraps this in.
+            if (accounting.ExaminedNothing)
+            {
+                string warning = accounting.ExaminedNothingWarning();
+                if (outputFormat == "github")
+                    Console.WriteLine($"::warning title=Compliance coverage::{warning}");
+                else
+                    Console.WriteLine($"WARNING: {warning}");
+            }
+        }
+
         if (verbose)
         {
             if (status == TestStatus.Error || status == TestStatus.Warning)
@@ -84,6 +106,7 @@ public static void Write(
                 ["summary"]         = summary,
                 ["text"]            = text,
                 ["annotationCount"] = annotations.Count,
+                ["coverage"]        = accounting?.ToPayload() ?? new Dictionary<string, object>(),
                 ["annotations"]     = annotations.Select(a => new Dictionary<string, object>
                 {
                     ["path"]             = a.FilePath,

tools/ComplianceRunner/tests/Compliance.Tests/Integration/ComplianceRunnerE2ETests.cs

@@ -97,12 +97,22 @@ public void JsonOutput_CheckType_MatchesInput()
     // ── GitHub Actions output format ──────────────────────────────────────────
 
     [Test]
-    public void GitHubOutput_ForPassingRun_ProducesNoAnnotationLines()
+    public void GitHubOutput_ForPassingRun_ProducesNoFindingAnnotations()
     {
-        // Filtered-out files produce no ::error/::warning lines.
+        // This previously asserted stdout was entirely empty. That silence was the defect: a run
+        // that examined nothing looked exactly like a run that examined everything. What a
+        // passing run must not produce is a *finding* annotation, which is what the name means
+        // and what is asserted now. The coverage output is not a finding and changes no verdict.
         var (exitCode, stdout) = RunnerFixture.Run("ComplianceRunner",
             "code", "--output", "github", "nonexistent.md");
-        Assert.That(exitCode, Is.EqualTo(0));
-        Assert.That(stdout.Trim(), Is.Empty);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(exitCode, Is.EqualTo(0));
+            Assert.That(stdout, Does.Not.Contain("::error"), "no findings, so no error annotations");
+            Assert.That(stdout, Does.Contain("Coverage: 0 of 1 file(s) examined"));
+            Assert.That(stdout, Does.Contain("::warning title=Compliance coverage::"),
+                "the one annotation a passing run may produce is the report that it examined nothing");
+        });
     }
 }