Put coverage collection behind a --coverage option

Collection is off until it is asked for, either with the flag or with
`coverage: enabled: true` in .zunit.yml, alongside the paths to measure:

  coverage:
    enabled: false
    paths: src

Paths may be separated by commas or spaces. The test directory is left
out unless exclusions are configured explicitly, since test files are
not the code under test. Asking for coverage without configuring any
paths is an error rather than an empty report, because there would
otherwise be no way to tell it apart from code which never ran.

Records are written to coverage.data in the output directory, and what
was executed is summarised after the results. The summary reports the
number of lines covered in each file and no percentage, since working
out how many lines could have run needs the line analyzer.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GTtYX8VzPrk4srd6xzURor

.gitignore

@@ -2,6 +2,10 @@
 /tests/_output/*
 !/tests/_output/.gitkeep
 
+# The same, for the fixture project the coverage tests run against
+/tests/_support/coverage/project/tests/_output/*
+!/tests/_support/coverage/project/tests/_output/.gitkeep
+
 # Ignore the compiled program
 /zunit
 

src/commands/init.zsh

@@ -50,7 +50,10 @@ directories:
   support: tests/_support
 time_limit: 0
 fail_fast: false
-allow_risky: false"
+allow_risky: false
+coverage:
+  enabled: false
+  paths: src"
 
   # An example test file
   local example="#!/usr/bin/env zunit

src/commands/run.zsh

@@ -19,6 +19,7 @@ function _zunit_run_usage() {
   echo "      --output-html      Print results to a HTML page"
   echo "      --allow-risky      Supress warnings generated for risky tests"
   echo "      --time-limit <n>   Set a time limit of n seconds for each test"
+  echo "      --coverage         Measure which lines of code the tests execute"
 }
 
 ###
@@ -55,6 +56,12 @@ function _zunit_output_results() {
   echo "$(color yellow '‼') Warnings    $warnings                 "
   echo
 
+  if [[ -n $coverage ]]; then
+    _zunit_coverage_summary
+    echo
+    echo "Coverage data written at $PWD/$logfile_coverage"
+  fi
+
   [[ -n $output_text ]] && echo "TAP report written at $PWD/$logfile_text"
   [[ -n $output_html ]] && echo "HTML report written at $PWD/$logfile_html"
 }
@@ -441,6 +448,7 @@ function _zunit_run() {
   local -a arguments testfiles
   local fail_fast tap allow_risky verbose
   local output_text logfile_text output_html logfile_html
+  local coverage logfile_coverage
 
   # Load the datetime module, and record the start time
   zmodload zsh/datetime
@@ -455,7 +463,8 @@ function _zunit_run() {
     -output-text=output_text \
     -output-html=output_html \
     -allow-risky=allow_risky \
-    -time-limit:=time_limit
+    -time-limit:=time_limit \
+    -coverage=coverage
 
   # TAP output is enabled
   if [[ -n $tap ]] || [[ "$zunit_config_tap" = "true" ]]; then
@@ -475,8 +484,13 @@ function _zunit_run() {
     echo
   fi
 
+  # Check if coverage is specified in the config or as an option
+  if [[ -z $coverage ]] && [[ "$zunit_config_coverage_enabled" = "true" ]]; then
+    coverage=1
+  fi
+
   # Text output has been requested
-  if [[ -n $output_text || -n $output_html ]]; then
+  if [[ -n $output_text || -n $output_html || -n $coverage ]]; then
     # Make sure we have a config file, otherwise we can't determine
     # which directory to write logs to
     if [[ $missing_config -eq 1 ]]; then
@@ -508,6 +522,21 @@ function _zunit_run() {
     _zunit_html_header > $logfile_html
   fi
 
+  if [[ -n $coverage ]]; then
+    # Without any paths to measure, coverage would silently record
+    # nothing at all, so say so rather than reporting an empty result
+    if ! _zunit_coverage_configure; then
+      echo $(color red 'Coverage paths must be specified in .zunit.yml') >&2
+      echo 'e.g.' >&2
+      echo '  coverage:' >&2
+      echo '    paths: src' >&2
+      exit 1
+    fi
+
+    logfile_coverage="$zunit_config_directories_output/coverage.data"
+    _zunit_coverage_init "$logfile_coverage" || exit 1
+  fi
+
   if [[ -n $zunit_config_directories_support ]]; then
     # Check that the support directory exists
     local support="$zunit_config_directories_support"
@@ -580,6 +609,13 @@ function _zunit_run() {
 
   end_time=$((EPOCHREALTIME*1000))
 
+  # Close the coverage datafile and read the records back before
+  # anything is reported
+  if [[ -n $coverage ]]; then
+    _zunit_coverage_finish
+    _zunit_coverage_parse
+  fi
+
   # Print report footers
   [[ -n $tap ]] && echo "1..$total"
   [[ -n $output_text ]] && echo "1..$total" >> $logfile_text

src/coverage.zsh

@@ -129,6 +129,65 @@ function _zunit_coverage_includes() {
   return 1
 }
 
+###
+# Work out which paths coverage should be measured within, from the
+# values in .zunit.yml. Returns non-zero if there is nothing to measure
+###
+function _zunit_coverage_configure() {
+  local value
+
+  zunit_coverage_paths=()
+  zunit_coverage_exclude=()
+
+  # Values arrive from the config parser as arrays, and each of them
+  # may list more than one path, separated by commas or spaces
+  for value in "${zunit_config_coverage_paths[@]}"; do
+    zunit_coverage_paths+=(${=${value//,/ }})
+  done
+
+  for value in "${zunit_config_coverage_exclude[@]}"; do
+    zunit_coverage_exclude+=(${=${value//,/ }})
+  done
+
+  # Test files are not the code under test, so the test directory is
+  # left out by default. Configuring exclusions replaces this, rather
+  # than adding to it, so that measuring tests remains possible
+  if (( ! ${#zunit_coverage_exclude} )) && [[ -n $zunit_config_directories_tests ]]; then
+    zunit_coverage_exclude=("$zunit_config_directories_tests")
+  fi
+
+  (( ${#zunit_coverage_paths} ))
+}
+
+###
+# Print the number of covered lines in each file which was measured
+###
+function _zunit_coverage_summary() {
+  local key file
+  local -A files
+
+  for key in "${(k)zunit_coverage_hits[@]}"; do
+    file="${key%:*}"
+    files[$file]=$(( ${files[$file]:-0} + 1 ))
+  done
+
+  echo
+  echo "$(color yellow underline 'Coverage')                        "
+
+  if (( ! ${#files} )); then
+    echo "$(color yellow 'No lines were recorded within the configured paths')"
+    return 0
+  fi
+
+  # Until the line analyzer lands there is no denominator to report a
+  # percentage against, so report what was executed and nothing more
+  for file in "${(@ok)files}"; do
+    printf '%-52s %6d lines\n' "${file#$PWD/}" "$files[$file]"
+  done
+
+  return 0
+}
+
 ###
 # Read a datafile of coverage records into $zunit_coverage_hits
 ###

src/zunit.zsh

@@ -27,6 +27,7 @@ function _zunit_usage() {
   echo "      --output-html  Print results to a HTML page"
   echo "      --allow-risky  Supress warnings generated for risky tests"
   echo "      --time-limit   Set a time limit in seconds for each test"
+  echo "      --coverage     Measure which lines of code the tests execute"
 }
 
 ###

tests/_support/coverage/project/.zunit.yml

@@ -0,0 +1,11 @@
+tap: false
+directories:
+  tests: tests
+  output: tests/_output
+time_limit: 15
+fail_fast: false
+allow_risky: false
+verbose: false
+coverage:
+  enabled: false
+  paths: src

tests/_support/coverage/project/src/calculator.zsh

@@ -0,0 +1,9 @@
+function calculator_add() {
+  local a=$1 b=$2
+  echo $(( a + b ))
+}
+
+function calculator_subtract() {
+  local a=$1 b=$2
+  echo $(( a - b ))
+}

tests/_support/coverage/project/tests/calculator.zunit

@@ -0,0 +1,9 @@
+#!/usr/bin/env zunit
+
+@test 'Test adding two numbers' {
+  load ../src/calculator
+
+  run calculator_add 2 3
+
+  assert $output same_as '5'
+}

tests/coverage/options.zunit

@@ -0,0 +1,100 @@
+#!/usr/bin/env zunit
+
+@setup {
+  zunit_bin="$PWD/zunit"
+  coverage_project="$PWD/tests/_support/coverage/project"
+
+  # Each test works against its own copy of the fixture project, so
+  # that it can rewrite the config without disturbing anything else
+  coverage_tmp="$PWD/tests/_output/coverage-project"
+  rm -rf "$coverage_tmp"
+  cp -R "$coverage_project" "$coverage_tmp"
+}
+
+@teardown {
+  rm -rf "$coverage_tmp"
+}
+
+@test 'Test the --coverage option reports covered lines' {
+  evl cd "$coverage_tmp" '&&' "$zunit_bin" run --coverage
+
+  assert $state equals 0
+  assert "$output" contains 'Coverage'
+  assert "$output" contains 'src/calculator.zsh'
+}
+
+@test 'Test coverage is not reported unless it is asked for' {
+  evl cd "$coverage_tmp" '&&' "$zunit_bin" run
+
+  assert $state equals 0
+  assert "$output" does_not_contain 'Coverage'
+}
+
+@test 'Test coverage can be enabled in the config file' {
+  sed -i.bak 's/enabled: false/enabled: true/' "$coverage_tmp/.zunit.yml"
+
+  evl cd "$coverage_tmp" '&&' "$zunit_bin" run
+
+  assert $state equals 0
+  assert "$output" contains 'src/calculator.zsh'
+}
+
+@test 'Test coverage without any configured paths fails' {
+  sed -i.bak '/paths:/d' "$coverage_tmp/.zunit.yml"
+
+  evl cd "$coverage_tmp" '&&' "$zunit_bin" run --coverage
+
+  assert $state equals 1
+  assert "$output" contains 'Coverage paths must be specified'
+}
+
+@test 'Test coverage writes a datafile' {
+  evl cd "$coverage_tmp" '&&' "$zunit_bin" run --coverage
+
+  assert "$coverage_tmp/tests/_output/coverage.data" is_file
+}
+
+@test 'Test coverage paths may be separated by commas' {
+  zunit_config_coverage_paths=('src, lib')
+  zunit_config_coverage_exclude=()
+
+  _zunit_coverage_configure
+
+  assert "${#zunit_coverage_paths}" equals 2
+  assert 'src' in ${zunit_coverage_paths[@]}
+  assert 'lib' in ${zunit_coverage_paths[@]}
+}
+
+@test 'Test coverage excludes the test directory by default' {
+  zunit_config_coverage_paths=('src')
+  zunit_config_coverage_exclude=()
+  zunit_config_directories_tests='tests'
+
+  _zunit_coverage_configure
+
+  assert 'tests' in ${zunit_coverage_exclude[@]}
+}
+
+@test 'Test configured coverage exclusions replace the default' {
+  zunit_config_coverage_paths=('src')
+  zunit_config_coverage_exclude=('src/vendor')
+  zunit_config_directories_tests='tests'
+
+  _zunit_coverage_configure
+
+  assert "${#zunit_coverage_exclude}" equals 1
+  assert 'src/vendor' in ${zunit_coverage_exclude[@]}
+}
+
+@test 'Test coverage configuration fails without any paths' {
+  local state
+
+  zunit_config_coverage_paths=()
+  zunit_config_coverage_exclude=()
+
+  _zunit_coverage_configure && state=$? || state=$?
+
+  assert $state equals 1
+}
+
+# vim:ft=zsh:et:sts=2:sw=2

zunit.zsh-completion

@@ -25,7 +25,8 @@ _zunit() {
     '--output-text[print results to a text log, in TAP compatible format]' \
     '--output-html[print results to a HTML page]' \
     '--allow-risky[supress warnings generated for risky tests]' \
-    '--time-limit[set a time limit in seconds for each test]'
+    '--time-limit[set a time limit in seconds for each test]' \
+    '--coverage[measure which lines of code the tests execute]'
 
   case "$state" in
     args )
@@ -47,7 +48,8 @@ _zunit() {
             '--output-text[print results to a text log, in TAP compatible format]' \
             '--output-html[print results to a HTML page]' \
             '--allow-risky[supress warnings generated for risky tests]' \
-            '--time-limit[set a time limit in seconds for each test]'
+            '--time-limit[set a time limit in seconds for each test]' \
+            '--coverage[measure which lines of code the tests execute]'
 
           _arguments \
             '*:tests:_files'