feat: add quiet test output mode

README.md

@@ -202,6 +202,8 @@ Use `--engine` (or `EXODUS_TEST_ENGINE=`) to specify one of:
 
 - `--watch` — operate in watch mode and re-run tests on file changes
 
+- `--quiet` — only report failing tests. The final summary is still printed
+
 - `--only` — only run the tests marked with `test.only`
 
 - `--passWithNoTests` — do not error when no test files were found

bin/index.js

@@ -97,6 +97,7 @@ function parseOptions() {
     coverage: getEnvFlag('EXODUS_TEST_COVERAGE'),
     coverageEngine: process.platform === 'win32' ? 'node' : 'c8', // c8 or node. TODO: can we use c8 on win?
     watch: false,
+    quiet: false,
     only: false,
     passWithNoTests: false,
     writeSnapshots: false,
@@ -185,6 +186,9 @@ function parseOptions() {
       case '--watch':
         options.watch = true
         break
+      case '--quiet':
+        options.quiet = true
+        break
       case '--test-only':
       case '--only':
         options.only = true
@@ -295,6 +299,7 @@ setEnv('EXODUS_TEST_ENGINE', options.engine) // e.g. 'hermes:bundle', 'node:bund
 setEnv('EXODUS_TEST_PLATFORM', options.binary === 'shermes' ? 'hermes' : options.binary) // e.g. 'hermes', 'node'
 setEnv('EXODUS_TEST_TIMEOUT', options.testTimeout)
 setEnv('EXODUS_TEST_DEVTOOLS', options.devtools ? '1' : '')
+process.env.EXODUS_TEST_QUIET = options.quiet ? '1' : '' // internal signal for the reporter
 setEnv('EXODUS_TEST_IS_BROWSER', isBrowserLike ? '1' : '')
 setEnv('EXODUS_TEST_IS_BAREBONE', options.barebone ? '1' : '')
 setEnv('EXODUS_TEST_ENVIRONMENT', options.bundle ? 'bundle' : '') // perhaps switch to _IS_BUNDLED?
@@ -805,17 +810,31 @@ const mainWorker :Workerd.Worker = (
   }
 
   const { format, head, middle, tail, timeLabel, summary } = await import('./reporter.js')
+  const identity = (value) => value
+  const passOrSkipLine = /^(✔ PASS|⏭ SKIP) /u
+  // In quiet mode, drop passing/skipped lines from a suite's captured output (failures are kept).
+  const filterQuietOutput = options.quiet
+    ? (chunk) =>
+        chunk
+          .split('\n')
+          .filter((line) => !passOrSkipLine.test(line))
+          .join('\n')
+    : identity
 
   const failures = []
   const tasks = files.map((file) => ({ file, task: runConcurrent(file) }))
   console.time(timeLabel)
   for (const { file, task } of tasks) {
-    head(file)
     const { ok, output, ms } = await task
+    if (!ok) failures.push(file)
+    if (options.quiet && ok) continue // quiet mode: only surface failing suites
+    head(file)
     middle(file, ok, ms)
-    for (const chunk of output.filter((x) => x.trim())) console.log(format(chunk).trimEnd())
+    for (const chunk of output.map(filterQuietOutput).filter((x) => x.trim())) {
+      console.log(format(chunk).trimEnd())
+    }
+
     tail(file)
-    if (!ok) failures.push(file)
   }
 
   if (failures.length > 0) process.exitCode = 1

bin/reporter.js

@@ -26,13 +26,17 @@ export const format = (chunk) => {
 
 const formatTime = (ms) => (ms ? color(` (${ms}ms)`, dim) : '')
 const formatSuffix = (d) => `${formatTime(d.details.duration_ms)}${d.todo ? ' # TODO' : ''}`
+const isNodeTestSummaryDiagnostic = (message) =>
+  /^(suites|tests|pass|fail|cancelled|skipped|todo) \d+$/.test(message) ||
+  /^duration_ms \d+(?:\.\d+)?$/.test(message)
 
 const cwd = process.cwd()
 const INBAND_PREFIX = 'EXODUS_TEST_INBAND:'
 const inbandFileAbsolute = fileURLToPath(import.meta.resolve('./inband.js'))
 const inbandFile = relative(cwd, inbandFileAbsolute)
 
 const groupCI = CI && !process.execArgv.includes('--watch') && !LERNA_PACKAGE_NAME // lerna+nx groups already
+const quiet = process.env.EXODUS_TEST_QUIET === '1'
 export const timeLabel = color('Total time', dim)
 const filename = (f) => (f === inbandFile || f === inbandFileAbsolute ? 'In-band tests' : f)
 export const head = groupCI ? () => {} : (file) => console.log(color(`# ${filename(file)}`, 'bold'))
@@ -100,9 +104,24 @@ export default async function nodeTestReporterExodus(source) {
   })
 
   const log = []
-  const print = (msg) => (groupCI ? log.push(msg) : console.log(msg))
+  const buffered = groupCI || quiet
+  const print = (msg) => (buffered ? log.push(msg) : console.log(msg))
+  const dumpDiagnostics = () => {
+    if (!quiet || failedFiles.size > 0) {
+      for (const line of diagnostic) console.log(line)
+    }
+
+    diagnostic.length = 0
+  }
+
   const dump = () => {
-    middle(file, !failedFiles.has(file))
+    const ok = !failedFiles.has(file)
+    if (quiet && ok) {
+      log.length = 0
+      return
+    }
+
+    middle(file, ok)
     for (const line of log) console.log(line)
     log.length = 0
     tail()
@@ -114,6 +133,14 @@ export default async function nodeTestReporterExodus(source) {
   let file
   const diagnostic = []
   const delayed = []
+  const finishWatchCycle = () => {
+    if (file !== undefined) dump()
+    dumpDiagnostics()
+    delayed.length = 0
+    failedFiles.clear()
+    file = undefined
+  }
+
   const isTopLevelTest = ({ nesting, line, column, name, file }) =>
     nesting === 0 && line === 1 && column === 1 && file.endsWith(name) && resolve(name) === file // some events have data.file resolved, some not)
   const processNewFile = (data) => {
@@ -123,7 +150,9 @@ export default async function nodeTestReporterExodus(source) {
     if (file !== undefined) dump()
     file = newFile
     assert(files.has(file), 'Cound not determine file')
-    head(file)
+    // quiet (non-CI): buffer the header so it only prints for failing suites (under CI, middle() emits it)
+    if (quiet && !groupCI) log.push(color(`# ${filename(file)}`, 'bold'))
+    else head(file)
   }
 
   const pathstr = (p) => (p[0]?.startsWith(INBAND_PREFIX) ? p.slice(1) : p).join(' > ')
@@ -149,8 +178,11 @@ export default async function nodeTestReporterExodus(source) {
         while (delayed.length > 0) print(delayed.shift())
         break
       case 'test:pass':
-        const label = data.skip ? color('⏭ SKIP ', dim) : color('✔ PASS ', 'green')
-        if (!pskip(path)) print(`${label}${pathstr(path)}${formatSuffix(data)}`)
+        if (!quiet) {
+          const label = data.skip ? color('⏭ SKIP ', dim) : color('✔ PASS ', 'green')
+          if (!pskip(path)) print(`${label}${pathstr(path)}${formatSuffix(data)}`)
+        }
+
         assert(path.pop() === data.name)
         break
       case 'test:fail':
@@ -170,10 +202,12 @@ export default async function nodeTestReporterExodus(source) {
         break
       case 'test:watch:drained':
         assert(!groupCI, 'Can not mix --watch with CI grouping')
+        if (quiet) finishWatchCycle()
         console.log(color(`ℹ waiting for changes as we are in --watch mode`, 'blue'))
         break
       case 'test:diagnostic':
         if (/^suites \d+$/.test(data.message)) break // we count suites = files
+        if (quiet && isNodeTestSummaryDiagnostic(data.message)) break // summary() prints the result
         diagnostic.push(color(`ℹ ${data.message}`, 'blue'))
         break
       case 'test:stderr':
@@ -188,7 +222,11 @@ export default async function nodeTestReporterExodus(source) {
   }
 
   dump()
-  for (const line of delayed) console.log(line)
-  for (const line of diagnostic) console.log(line)
+  if (!quiet) {
+    for (const line of delayed) console.log(line)
+  }
+
+  dumpDiagnostics()
+
   summary([...files], [...failedFiles])
 }