Update to v2

.dockerignore

@@ -1,8 +1,3 @@
-.github/
-tests/
-.appends
-.git
-.gitignore
-.gitattributes
-.dockerignore
-Dockerfile
+*
+!bin/
+!lib/

Dockerfile

@@ -1,9 +1,10 @@
 FROM alpine:3.23.5@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
 
-RUN apk add --no-cache git bash jq vim && \
-    rm -rf /var/cache/apk/*
+ARG VADER_REV=429b669e6158be3a9fc110799607c232e6ed8e29
 
-RUN git clone https://github.com/junegunn/vader.vim.git /opt/test-runner/vader.vim
+RUN apk add --no-cache bash git jq vim \
+    && git clone --revision="$VADER_REV" --depth=1 https://github.com/junegunn/vader.vim.git /opt/vader.vim \
+    && rm -rf /opt/vader.vim/.git
 
 COPY . /opt/test-runner
 WORKDIR /opt/test-runner

bin/run-tests.sh

@@ -12,25 +12,28 @@
 # ./bin/run-tests.sh
 
 exit_code=0
+temp_dir=$(mktemp -d) || exit 1
+trap 'rm -rf "${temp_dir}"' EXIT
 
-# Iterate over all test directories
-for test_dir in tests/*; do
-    test_dir_name=$(basename "${test_dir}")
-    test_dir_path=$(realpath "${test_dir}")
+run_test() {
+    test_dir_name=$(basename "${1}")
+    test_dir_path=$(realpath "${1}")
     results_file_path="${test_dir_path}/results.json"
     expected_results_file_path="${test_dir_path}/expected_results.json"
+    formatted_results_file_path="${temp_dir}/${test_dir_name}-results.json"
+    formatted_expected_results_file_path="${temp_dir}/${test_dir_name}-expected-results.json"
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
-    # Normalize the results file
-    sed -i -E \
-        -e 's/Elapsed time: [0-9]+\.[0-9]+ sec\.//' \
-        -e 's~/tmp/[^/]+/[^,]+,\s*~~g' \
-        -e "s~${test_dir_path}~/solution~g" \
-        "${results_file_path}"
+    jq -S . "${results_file_path}" > "${formatted_results_file_path}"
+    jq -S . "${expected_results_file_path}" > "${formatted_expected_results_file_path}"
 
     echo "${test_dir_name}: comparing results.json to expected_results.json"
-    diff "${results_file_path}" "${expected_results_file_path}"
+    diff "${formatted_results_file_path}" "${formatted_expected_results_file_path}"
+}
+
+for test_dir in tests/*; do
+    run_test "${test_dir}"
 
     if [ $? -ne 0 ]; then
         exit_code=1

bin/run.sh

@@ -15,7 +15,7 @@
 # Example:
 # ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
-# If any required arguments is missing, print the usage and exit
+# If any required argument is missing, print the usage and exit.
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/"
     exit 1
@@ -25,33 +25,18 @@ slug="$1"
 input_dir="${2%/}"
 output_dir="${3%/}"
 exercise="${slug//-/_}"
-export implementation_file="${input_dir}/${exercise}.vim"
+implementation_file="${input_dir}/${exercise}.vim"
 tests_file="${input_dir}/${exercise}.vader"
 results_file="${output_dir}/results.json"
 
-# Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
+vim -Nu NONE -i NONE -n -es -S lib/runner.vim -- \
+    "${implementation_file}" "${tests_file}" "${results_file}" "${input_dir}" /opt/vader.vim
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(TERM=xterm HOME=/dev/null vim -XNu .vimrc -i NONE -c ":so $implementation_file" -c "Vader! ${tests_file}" 2>&1)
-
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
-if [ $? -eq 0 ] && [ "${test_output}" != "[EXECUTE] (X)" ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
-    # Strip off the logging information
-    sanitized_test_output=$(printf "${test_output}" | sed -n '/Starting Vader/,$p')
-
-    # Manually add colors to the output to help scanning the output for errors
-    colorized_test_output=$(echo "${sanitized_test_output}" \
-         | GREP_COLOR='01;31' grep --color=always -E -e '.*\[EXECUTE\] \(X\).*|$' \
-         | GREP_COLOR='01;32' grep --color=always -E -e '.*\[EXECUTE\].*|$')
-
-    jq -n --arg output "${colorized_test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+if [ ! -s "${results_file}" ]; then
+    jq -n '{version: 2, status: "error", message: "The Vim test runner could not start. Please start a thread on the Exercism forums.", tests: []}' > "${results_file}"
 fi
 
 echo "${slug}: done"

lib/runner.vim

@@ -0,0 +1,120 @@
+" Orchestrates a Vader run and writes Exercism test-runner v2 JSON.
+set nocompatible
+
+let s:arguments = v:argv[index(v:argv, '--') + 1 :]
+let s:solution_file = s:arguments[0]
+let s:test_file = s:arguments[1]
+let s:results_file = s:arguments[2]
+let s:input_dir = s:arguments[3]
+let s:vader_path = s:arguments[4]
+
+function! s:run() abort
+  try
+    if !filereadable(s:solution_file)
+      call s:error('Solution file not found: '.s:solution_file)
+      return
+    endif
+    if !filereadable(s:test_file)
+      call s:error('Test file not found: '.s:test_file)
+      return
+    endif
+    if !isdirectory(s:vader_path)
+      call s:error('Vader test framework is not installed.')
+      return
+    endif
+
+    execute 'set runtimepath^='.fnameescape(s:vader_path)
+    runtime plugin/vader.vim
+
+    execute 'source '.fnameescape(s:solution_file)
+    try
+      let cases = vader#parser#parse(s:test_file, 1, 0)
+    catch
+      call s:error('Vader test file syntax error.')
+      return
+    endtry
+
+    execute 'silent Vader '.fnameescape(s:test_file)
+    let report = split(get(g:, 'vader_report', ''), "\n")
+    let errors_by_line = {}
+    for vader_error in get(g:, 'vader_errors', [])
+      let errors_by_line[string(vader_error.lnum)] = vader_error
+    endfor
+
+    let tests = []
+    let all_errors = 1
+    let failed = 0
+    for case in cases
+      let test = {
+            \ 'name': s:case_name(case),
+            \ 'status': 'pass',
+            \ 'test_code': join(case.execute, "\n"),
+            \ }
+      let vader_error = get(errors_by_line, string(case.lnum), {})
+      if !empty(vader_error)
+        let message = s:failure_message(vader_error, report)
+        let test.status = s:failure_status(message)
+        let test.message = s:clean_message(message)
+      endif
+      let all_errors = all_errors && test.status ==# 'error'
+      let failed = failed || test.status !=# 'pass'
+      call add(tests, test)
+    endfor
+
+    if empty(tests)
+      call s:error('No Vader test cases were executed.')
+    elseif all_errors
+      call s:error(get(tests[0], 'message', 'All Vader test cases errored.'))
+    else
+      call s:write({
+            \ 'version': 2,
+            \ 'status': failed ? 'fail' : 'pass',
+            \ 'message': v:null,
+            \ 'tests': tests,
+            \ })
+    endif
+  catch
+    call s:error(v:exception)
+  endtry
+endfunction
+
+function! s:write(document) abort
+  call writefile([json_encode(a:document)], s:results_file)
+endfunction
+
+function! s:clean_message(message) abort
+  if !empty(s:input_dir)
+    return substitute(a:message, '\V'.escape(s:input_dir, '\\').'\m', '<solution-dir>', 'g')
+  endif
+  return a:message
+endfunction
+
+function! s:error(message) abort
+  call s:write({
+        \ 'version': 2,
+        \ 'status': 'error',
+        \ 'message': s:clean_message(a:message),
+        \ 'tests': [],
+        \ })
+endfunction
+
+function! s:case_name(case) abort
+  let name = get(a:case.comment, 'execute', '')
+  return empty(name) ? printf('Test at line %d', a:case.lnum) : name
+endfunction
+
+function! s:failure_message(vader_error, report) abort
+  let report_line = matchstr(get(a:vader_error, 'text', ''), '(#\zs\d\+\ze)')
+  if !empty(report_line) && report_line >= 1 && report_line <= len(a:report)
+    let message = a:report[report_line - 1]
+    return substitute(message, '^\s*(\d\+/\d\+) \[[^]]*\] (X) ', '', '')
+  endif
+  return get(a:vader_error, 'text', 'Test failed.')
+endfunction
+
+function! s:failure_status(message) abort
+  return a:message =~# '^Vim\%(([^)]*)\)\?:' ? 'error' : 'fail'
+endfunction
+
+call s:run()
+qa!

tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vader

@@ -0,0 +1,5 @@
+Execute (first error):
+  call MissingFunctionOne()
+
+Execute (second error):
+  call MissingFunctionTwo()

tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vim

@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction

tests/error-if-all-tests-raise-errors/expected_results.json

@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim(call):E117: Unknown function: MissingFunctionOne"
+}

tests/error-if-empty-solution/error_if_empty_solution.vader

@@ -0,0 +1,2 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)

tests/error-if-empty-solution/expected_results.json

@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim(call):E117: Unknown function: Identity"
+}

tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vader

@@ -0,0 +1,2 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)

tests/error-if-solution-syntax-error/expected_results.json

@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim:E492: Not an editor command: FUNCTIONNNNN @#@#!"
+}

tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vader

@@ -0,0 +1,5 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)
+
+Executte (returns 2):
+  AssertEqual 2, Identity(2)

tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vim

@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction

tests/error-if-vader-syntax-error/expected_results.json

@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vader test file syntax error."
+}

tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vader

@@ -0,0 +1,2 @@
+Execute (returns a value):
+  AssertEqual 1, answer

tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vim

@@ -0,0 +1 @@
+var answer = 1

tests/error-if-vim9script-has-no-header/expected_results.json

@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim(var):E1124: \":var\" cannot be used in legacy Vim script"
+}