From 2282bf5354324f6cb50aa60044983ac32eca9987 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 08:21:55 -0700
Subject: [PATCH 01/10] Pin Vader

---
 .dockerignore | 13 +++++--------
 Dockerfile    | 12 ++++++++----
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 9c9581c..ca330b6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,8 +1,5 @@
-.github/
-tests/
-.appends
-.git
-.gitignore
-.gitattributes
-.dockerignore
-Dockerfile
+*
+!bin/
+!bin/**
+!lib/
+!lib/**
diff --git a/Dockerfile b/Dockerfile
index bad5a3f..89f38c5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,13 @@
 FROM alpine:3.23.5@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
 
-RUN apk add --no-cache git bash jq vim && \
-    rm -rf /var/cache/apk/*
-
-RUN git clone https://github.com/junegunn/vader.vim.git /opt/test-runner/vader.vim
+ARG VADER_REV=429b669e6158be3a9fc110799607c232e6ed8e29
+
+RUN apk add --no-cache bash git jq vim \
+    && git init /opt/vader.vim \
+    && git -C /opt/vader.vim remote add origin https://github.com/junegunn/vader.vim.git \
+    && git -C /opt/vader.vim fetch --depth 1 origin "$VADER_REV" \
+    && git -C /opt/vader.vim checkout --detach FETCH_HEAD \
+    && rm -rf /opt/vader.vim/.git
 
 COPY . /opt/test-runner
 WORKDIR /opt/test-runner

From b7eb8cbb814fedbd0e0a1a1a68373a8221a13bf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 08:52:08 -0700
Subject: [PATCH 02/10] Add success tests

---
 .../expected_results.json                       | 17 +++++++++++++++++
 .../success_if_all_tests_pass.vader             |  6 ++++++
 .../success_if_all_tests_pass.vim               |  3 +++
 .../expected_results.json                       | 17 +++++++++++++++++
 .../success_if_all_vim9script_tests_pass.vader  |  5 +++++
 .../success_if_all_vim9script_tests_pass.vim    |  5 +++++
 .../expected_results.json                       | 17 +++++++++++++++++
 .../success_if_test_names_repeat.vader          |  5 +++++
 .../success_if_test_names_repeat.vim            |  3 +++
 9 files changed, 78 insertions(+)
 create mode 100644 tests/success-if-all-tests-pass/expected_results.json
 create mode 100644 tests/success-if-all-tests-pass/success_if_all_tests_pass.vader
 create mode 100644 tests/success-if-all-tests-pass/success_if_all_tests_pass.vim
 create mode 100644 tests/success-if-all-vim9script-tests-pass/expected_results.json
 create mode 100644 tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vader
 create mode 100644 tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vim
 create mode 100644 tests/success-if-test-names-repeat/expected_results.json
 create mode 100644 tests/success-if-test-names-repeat/success_if_test_names_repeat.vader
 create mode 100644 tests/success-if-test-names-repeat/success_if_test_names_repeat.vim

diff --git a/tests/success-if-all-tests-pass/expected_results.json b/tests/success-if-all-tests-pass/expected_results.json
new file mode 100644
index 0000000..8b5d05f
--- /dev/null
+++ b/tests/success-if-all-tests-pass/expected_results.json
@@ -0,0 +1,17 @@
+{
+  "status": "pass",
+  "version": 2,
+  "tests": [
+    {
+      "status": "pass",
+      "name": "returns 1",
+      "test_code": "AssertEqual 1, Identity(1)"
+    },
+    {
+      "status": "pass",
+      "name": "returns 2",
+      "test_code": "AssertEqual 2, Identity(2)"
+    }
+  ],
+  "message": null
+}
diff --git a/tests/success-if-all-tests-pass/success_if_all_tests_pass.vader b/tests/success-if-all-tests-pass/success_if_all_tests_pass.vader
new file mode 100644
index 0000000..131fe7e
--- /dev/null
+++ b/tests/success-if-all-tests-pass/success_if_all_tests_pass.vader
@@ -0,0 +1,6 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)
+
+Execute (returns 2):
+  AssertEqual 2, Identity(2)
+
diff --git a/tests/success-if-all-tests-pass/success_if_all_tests_pass.vim b/tests/success-if-all-tests-pass/success_if_all_tests_pass.vim
new file mode 100644
index 0000000..9476ada
--- /dev/null
+++ b/tests/success-if-all-tests-pass/success_if_all_tests_pass.vim
@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction
diff --git a/tests/success-if-all-vim9script-tests-pass/expected_results.json b/tests/success-if-all-vim9script-tests-pass/expected_results.json
new file mode 100644
index 0000000..8b5d05f
--- /dev/null
+++ b/tests/success-if-all-vim9script-tests-pass/expected_results.json
@@ -0,0 +1,17 @@
+{
+  "status": "pass",
+  "version": 2,
+  "tests": [
+    {
+      "status": "pass",
+      "name": "returns 1",
+      "test_code": "AssertEqual 1, Identity(1)"
+    },
+    {
+      "status": "pass",
+      "name": "returns 2",
+      "test_code": "AssertEqual 2, Identity(2)"
+    }
+  ],
+  "message": null
+}
diff --git a/tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vader b/tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vader
new file mode 100644
index 0000000..4fa49a6
--- /dev/null
+++ b/tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vader
@@ -0,0 +1,5 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)
+
+Execute (returns 2):
+  AssertEqual 2, Identity(2)
diff --git a/tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vim b/tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vim
new file mode 100644
index 0000000..0898a18
--- /dev/null
+++ b/tests/success-if-all-vim9script-tests-pass/success_if_all_vim9script_tests_pass.vim
@@ -0,0 +1,5 @@
+vim9script
+
+def g:Identity(value: number): number
+    return value
+enddef
diff --git a/tests/success-if-test-names-repeat/expected_results.json b/tests/success-if-test-names-repeat/expected_results.json
new file mode 100644
index 0000000..7f0deca
--- /dev/null
+++ b/tests/success-if-test-names-repeat/expected_results.json
@@ -0,0 +1,17 @@
+{
+  "status": "pass",
+  "version": 2,
+  "tests": [
+    {
+      "status": "pass",
+      "name": "duplicate name",
+      "test_code": "AssertEqual 2, Identity(2)"
+    },
+    {
+      "status": "pass",
+      "name": "duplicate name",
+      "test_code": "AssertEqual 1, Identity(1)"
+    }
+  ],
+  "message": null
+}
diff --git a/tests/success-if-test-names-repeat/success_if_test_names_repeat.vader b/tests/success-if-test-names-repeat/success_if_test_names_repeat.vader
new file mode 100644
index 0000000..90eff11
--- /dev/null
+++ b/tests/success-if-test-names-repeat/success_if_test_names_repeat.vader
@@ -0,0 +1,5 @@
+Execute (duplicate name):
+  AssertEqual 2, Identity(2)
+
+Execute (duplicate name):
+  AssertEqual 1, Identity(1)
diff --git a/tests/success-if-test-names-repeat/success_if_test_names_repeat.vim b/tests/success-if-test-names-repeat/success_if_test_names_repeat.vim
new file mode 100644
index 0000000..9476ada
--- /dev/null
+++ b/tests/success-if-test-names-repeat/success_if_test_names_repeat.vim
@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction

From ea25a722ed89bc9194377ab412b68e943a7b2a6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 09:12:26 -0700
Subject: [PATCH 03/10] Add error tests

---
 .../error_if_all_tests_raise_errors.vader                   | 5 +++++
 .../error_if_all_tests_raise_errors.vim                     | 3 +++
 tests/error-if-all-tests-raise-errors/expected_results.json | 6 ++++++
 tests/error-if-empty-solution/error_if_empty_solution.vader | 2 ++
 tests/error-if-empty-solution/error_if_empty_solution.vim   | 0
 tests/error-if-empty-solution/expected_results.json         | 6 ++++++
 .../error_if_solution_syntax_error.vader                    | 2 ++
 .../error_if_solution_syntax_error.vim                      | 1 +
 tests/error-if-solution-syntax-error/expected_results.json  | 6 ++++++
 .../error_if_vader_syntax_error.vader                       | 5 +++++
 .../error_if_vader_syntax_error.vim                         | 3 +++
 tests/error-if-vader-syntax-error/expected_results.json     | 6 ++++++
 .../error_if_vim9script_has_no_header.vader                 | 2 ++
 .../error_if_vim9script_has_no_header.vim                   | 1 +
 .../error-if-vim9script-has-no-header/expected_results.json | 6 ++++++
 15 files changed, 54 insertions(+)
 create mode 100644 tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vader
 create mode 100644 tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vim
 create mode 100644 tests/error-if-all-tests-raise-errors/expected_results.json
 create mode 100644 tests/error-if-empty-solution/error_if_empty_solution.vader
 create mode 100644 tests/error-if-empty-solution/error_if_empty_solution.vim
 create mode 100644 tests/error-if-empty-solution/expected_results.json
 create mode 100644 tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vader
 create mode 100644 tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vim
 create mode 100644 tests/error-if-solution-syntax-error/expected_results.json
 create mode 100644 tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vader
 create mode 100644 tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vim
 create mode 100644 tests/error-if-vader-syntax-error/expected_results.json
 create mode 100644 tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vader
 create mode 100644 tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vim
 create mode 100644 tests/error-if-vim9script-has-no-header/expected_results.json

diff --git a/tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vader b/tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vader
new file mode 100644
index 0000000..6978d85
--- /dev/null
+++ b/tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vader
@@ -0,0 +1,5 @@
+Execute (first error):
+  call MissingFunctionOne()
+
+Execute (second error):
+  call MissingFunctionTwo()
diff --git a/tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vim b/tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vim
new file mode 100644
index 0000000..9476ada
--- /dev/null
+++ b/tests/error-if-all-tests-raise-errors/error_if_all_tests_raise_errors.vim
@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction
diff --git a/tests/error-if-all-tests-raise-errors/expected_results.json b/tests/error-if-all-tests-raise-errors/expected_results.json
new file mode 100644
index 0000000..4b4236a
--- /dev/null
+++ b/tests/error-if-all-tests-raise-errors/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim(call):E117: Unknown function: MissingFunctionOne"
+}
diff --git a/tests/error-if-empty-solution/error_if_empty_solution.vader b/tests/error-if-empty-solution/error_if_empty_solution.vader
new file mode 100644
index 0000000..46e4af7
--- /dev/null
+++ b/tests/error-if-empty-solution/error_if_empty_solution.vader
@@ -0,0 +1,2 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)
diff --git a/tests/error-if-empty-solution/error_if_empty_solution.vim b/tests/error-if-empty-solution/error_if_empty_solution.vim
new file mode 100644
index 0000000..e69de29
diff --git a/tests/error-if-empty-solution/expected_results.json b/tests/error-if-empty-solution/expected_results.json
new file mode 100644
index 0000000..b179e39
--- /dev/null
+++ b/tests/error-if-empty-solution/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim(call):E117: Unknown function: Identity"
+}
diff --git a/tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vader b/tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vader
new file mode 100644
index 0000000..46e4af7
--- /dev/null
+++ b/tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vader
@@ -0,0 +1,2 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)
diff --git a/tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vim b/tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vim
new file mode 100644
index 0000000..1ad4b8f
--- /dev/null
+++ b/tests/error-if-solution-syntax-error/error_if_solution_syntax_error.vim
@@ -0,0 +1 @@
+FUNCTIONNNNN @#@#!
\ No newline at end of file
diff --git a/tests/error-if-solution-syntax-error/expected_results.json b/tests/error-if-solution-syntax-error/expected_results.json
new file mode 100644
index 0000000..da66c87
--- /dev/null
+++ b/tests/error-if-solution-syntax-error/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim:E492: Not an editor command: FUNCTIONNNNN @#@#!"
+}
diff --git a/tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vader b/tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vader
new file mode 100644
index 0000000..c1b595b
--- /dev/null
+++ b/tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vader
@@ -0,0 +1,5 @@
+Execute (returns 1):
+  AssertEqual 1, Identity(1)
+
+Executte (returns 2):
+  AssertEqual 2, Identity(2)
diff --git a/tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vim b/tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vim
new file mode 100644
index 0000000..9476ada
--- /dev/null
+++ b/tests/error-if-vader-syntax-error/error_if_vader_syntax_error.vim
@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction
diff --git a/tests/error-if-vader-syntax-error/expected_results.json b/tests/error-if-vader-syntax-error/expected_results.json
new file mode 100644
index 0000000..d14ccff
--- /dev/null
+++ b/tests/error-if-vader-syntax-error/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vader test file syntax error."
+}
diff --git a/tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vader b/tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vader
new file mode 100644
index 0000000..08cf1fb
--- /dev/null
+++ b/tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vader
@@ -0,0 +1,2 @@
+Execute (returns a value):
+  AssertEqual 1, answer
diff --git a/tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vim b/tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vim
new file mode 100644
index 0000000..4cfb5c2
--- /dev/null
+++ b/tests/error-if-vim9script-has-no-header/error_if_vim9script_has_no_header.vim
@@ -0,0 +1 @@
+var answer = 1
diff --git a/tests/error-if-vim9script-has-no-header/expected_results.json b/tests/error-if-vim9script-has-no-header/expected_results.json
new file mode 100644
index 0000000..701f6bd
--- /dev/null
+++ b/tests/error-if-vim9script-has-no-header/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "status": "error",
+  "version": 2,
+  "tests": [],
+  "message": "Vim(var):E1124: \":var\" cannot be used in legacy Vim script"
+}

From 7e73623df799303d23545cd235e264b89446d8ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 17:25:09 -0700
Subject: [PATCH 04/10] Add failure tests

---
 .../expected_results.json                      | 18 ++++++++++++++++++
 .../failure_if_a_test_errors.vader             |  5 +++++
 .../failure_if_a_test_errors.vim               |  3 +++
 .../expected_results.json                      | 18 ++++++++++++++++++
 .../failure_if_some_tests_fail.vader           |  5 +++++
 .../failure_if_some_tests_fail.vim             |  3 +++
 .../expected_results.json                      | 18 ++++++++++++++++++
 ...failure_if_some_vim9script_tests_fail.vader |  5 +++++
 .../failure_if_some_vim9script_tests_fail.vim  |  5 +++++
 9 files changed, 80 insertions(+)
 create mode 100644 tests/failure-if-a-test-errors/expected_results.json
 create mode 100644 tests/failure-if-a-test-errors/failure_if_a_test_errors.vader
 create mode 100644 tests/failure-if-a-test-errors/failure_if_a_test_errors.vim
 create mode 100644 tests/failure-if-some-tests-fail/expected_results.json
 create mode 100644 tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vader
 create mode 100644 tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vim
 create mode 100644 tests/failure-if-some-vim9script-tests-fail/expected_results.json
 create mode 100644 tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vader
 create mode 100644 tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vim

diff --git a/tests/failure-if-a-test-errors/expected_results.json b/tests/failure-if-a-test-errors/expected_results.json
new file mode 100644
index 0000000..8f12d9a
--- /dev/null
+++ b/tests/failure-if-a-test-errors/expected_results.json
@@ -0,0 +1,18 @@
+{
+  "status": "fail",
+  "version": 2,
+  "tests": [
+    {
+      "status": "error",
+      "name": "unexpected error",
+      "message": "Vim(call):E117: Unknown function: MissingFunction",
+      "test_code": "call MissingFunction()"
+    },
+    {
+      "status": "pass",
+      "name": "returns 2",
+      "test_code": "AssertEqual 2, Identity(2)"
+    }
+  ],
+  "message": null
+}
diff --git a/tests/failure-if-a-test-errors/failure_if_a_test_errors.vader b/tests/failure-if-a-test-errors/failure_if_a_test_errors.vader
new file mode 100644
index 0000000..378617e
--- /dev/null
+++ b/tests/failure-if-a-test-errors/failure_if_a_test_errors.vader
@@ -0,0 +1,5 @@
+Execute (unexpected error):
+  call MissingFunction()
+
+Execute (returns 2):
+  AssertEqual 2, Identity(2)
diff --git a/tests/failure-if-a-test-errors/failure_if_a_test_errors.vim b/tests/failure-if-a-test-errors/failure_if_a_test_errors.vim
new file mode 100644
index 0000000..9476ada
--- /dev/null
+++ b/tests/failure-if-a-test-errors/failure_if_a_test_errors.vim
@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction
diff --git a/tests/failure-if-some-tests-fail/expected_results.json b/tests/failure-if-some-tests-fail/expected_results.json
new file mode 100644
index 0000000..82e3b41
--- /dev/null
+++ b/tests/failure-if-some-tests-fail/expected_results.json
@@ -0,0 +1,18 @@
+{
+  "status": "fail",
+  "version": 2,
+  "tests": [
+    {
+      "status": "fail",
+      "name": "fails",
+      "message": "1 should be equal to 2",
+      "test_code": "AssertEqual 2, Identity(1)"
+    },
+    {
+      "status": "pass",
+      "name": "passes",
+      "test_code": "AssertEqual 2, Identity(2)"
+    }
+  ],
+  "message": null
+}
diff --git a/tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vader b/tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vader
new file mode 100644
index 0000000..25d3a69
--- /dev/null
+++ b/tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vader
@@ -0,0 +1,5 @@
+Execute (fails):
+  AssertEqual 2, Identity(1)
+
+Execute (passes):
+  AssertEqual 2, Identity(2)
diff --git a/tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vim b/tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vim
new file mode 100644
index 0000000..9476ada
--- /dev/null
+++ b/tests/failure-if-some-tests-fail/failure_if_some_tests_fail.vim
@@ -0,0 +1,3 @@
+function! Identity(value) abort
+  return a:value
+endfunction
diff --git a/tests/failure-if-some-vim9script-tests-fail/expected_results.json b/tests/failure-if-some-vim9script-tests-fail/expected_results.json
new file mode 100644
index 0000000..82e3b41
--- /dev/null
+++ b/tests/failure-if-some-vim9script-tests-fail/expected_results.json
@@ -0,0 +1,18 @@
+{
+  "status": "fail",
+  "version": 2,
+  "tests": [
+    {
+      "status": "fail",
+      "name": "fails",
+      "message": "1 should be equal to 2",
+      "test_code": "AssertEqual 2, Identity(1)"
+    },
+    {
+      "status": "pass",
+      "name": "passes",
+      "test_code": "AssertEqual 2, Identity(2)"
+    }
+  ],
+  "message": null
+}
diff --git a/tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vader b/tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vader
new file mode 100644
index 0000000..25d3a69
--- /dev/null
+++ b/tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vader
@@ -0,0 +1,5 @@
+Execute (fails):
+  AssertEqual 2, Identity(1)
+
+Execute (passes):
+  AssertEqual 2, Identity(2)
diff --git a/tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vim b/tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vim
new file mode 100644
index 0000000..0898a18
--- /dev/null
+++ b/tests/failure-if-some-vim9script-tests-fail/failure_if_some_vim9script_tests_fail.vim
@@ -0,0 +1,5 @@
+vim9script
+
+def g:Identity(value: number): number
+    return value
+enddef

From 1a6e24a8f7e6d3df5c675644b117120c31cbcf14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 17:25:52 -0700
Subject: [PATCH 05/10] Delete v1 tests

---
 tests/example-all-fail/example_all_fail.vader | 13 ---------
 tests/example-all-fail/example_all_fail.vim   |  3 --
 tests/example-all-fail/expected_results.json  |  5 ----
 .../example_empty_file.vader                  | 28 -------------------
 .../example-empty-file/example_empty_file.vim |  0
 .../example-empty-file/expected_results.json  |  5 ----
 .../example_partial_fail.vader                | 28 -------------------
 .../example_partial_fail.vim                  |  3 --
 .../expected_results.json                     |  5 ----
 tests/example-success/example_success.vader   | 28 -------------------
 tests/example-success/example_success.vim     |  3 --
 tests/example-success/expected_results.json   |  4 ---
 .../example_syntax_error.vader                | 28 -------------------
 .../example_syntax_error.vim                  |  1 -
 .../expected_results.json                     |  5 ----
 .../example_vim9script.vader                  |  3 --
 .../example-vim9script/example_vim9script.vim |  7 -----
 .../example-vim9script/expected_results.json  |  4 ---
 18 files changed, 173 deletions(-)
 delete mode 100644 tests/example-all-fail/example_all_fail.vader
 delete mode 100644 tests/example-all-fail/example_all_fail.vim
 delete mode 100644 tests/example-all-fail/expected_results.json
 delete mode 100644 tests/example-empty-file/example_empty_file.vader
 delete mode 100644 tests/example-empty-file/example_empty_file.vim
 delete mode 100644 tests/example-empty-file/expected_results.json
 delete mode 100644 tests/example-partial-fail/example_partial_fail.vader
 delete mode 100644 tests/example-partial-fail/example_partial_fail.vim
 delete mode 100644 tests/example-partial-fail/expected_results.json
 delete mode 100644 tests/example-success/example_success.vader
 delete mode 100644 tests/example-success/example_success.vim
 delete mode 100644 tests/example-success/expected_results.json
 delete mode 100644 tests/example-syntax-error/example_syntax_error.vader
 delete mode 100644 tests/example-syntax-error/example_syntax_error.vim
 delete mode 100644 tests/example-syntax-error/expected_results.json
 delete mode 100644 tests/example-vim9script/example_vim9script.vader
 delete mode 100644 tests/example-vim9script/example_vim9script.vim
 delete mode 100644 tests/example-vim9script/expected_results.json

diff --git a/tests/example-all-fail/example_all_fail.vader b/tests/example-all-fail/example_all_fail.vader
deleted file mode 100644
index 0cede75..0000000
--- a/tests/example-all-fail/example_all_fail.vader
+++ /dev/null
@@ -1,13 +0,0 @@
-"
-" Version: 1.4.0
-"
-
-Execute (year not divisible by 4: common year):
-  let year = 2015
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 4, not divisible by 100: leap year):
-  let year = 1996
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
diff --git a/tests/example-all-fail/example_all_fail.vim b/tests/example-all-fail/example_all_fail.vim
deleted file mode 100644
index e666234..0000000
--- a/tests/example-all-fail/example_all_fail.vim
+++ /dev/null
@@ -1,3 +0,0 @@
-function! LeapYear(year) abort
-  return a:year % 2 == 1
-endfunction
diff --git a/tests/example-all-fail/expected_results.json b/tests/example-all-fail/expected_results.json
deleted file mode 100644
index bba9f64..0000000
--- a/tests/example-all-fail/expected_results.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "version": 1,
-  "status": "fail",
-  "message": "Starting Vader: 1 suite(s), 2 case(s)\n  Starting Vader: /solution/example_all_fail.vader\n    (1/2) [EXECUTE] year not divisible by 4: common year\n    (1/2) [EXECUTE] (X) 1 should be equal to 0\n    (2/2) [EXECUTE] year divisible by 4, not divisible by 100: leap year\n    (2/2) [EXECUTE] (X) 0 should be equal to 1\n  Success/Total: 0/2\nSuccess/Total: 0/2 (assertions: 0/2)\n\n\u001b[?1049h\u001b[22;0;0t\u001b[>4;2m\u001b[?1h\u001b=\u001b[?2004h\u001b[?1004h\u001b[24;1H\u001b[?2004l\u001b[>4;m\u001b[?2004h\u001b[>4;2m\u001b[?2004l\u001b[>4;m\u001b[23;2t\u001b[23;1t\u001b[?1004l\u001b[?2004l\u001b[?1l\u001b>\u001b[?1049l\u001b[23;0;0t\u001b[>4;m"
-}
diff --git a/tests/example-empty-file/example_empty_file.vader b/tests/example-empty-file/example_empty_file.vader
deleted file mode 100644
index ebc8488..0000000
--- a/tests/example-empty-file/example_empty_file.vader
+++ /dev/null
@@ -1,28 +0,0 @@
-"
-" Version: 1.4.0
-"
-
-Execute (year not divisible by 4: common year):
-  let year = 2015
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 4, not divisible by 100: leap year):
-  let year = 1996
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 100, not divisible by 400: common year):
-  let year = 2100
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 400: leap year):
-  let year = 2000
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 200, not divisible by 400: common year):
-  let year = 1800
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
diff --git a/tests/example-empty-file/example_empty_file.vim b/tests/example-empty-file/example_empty_file.vim
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/example-empty-file/expected_results.json b/tests/example-empty-file/expected_results.json
deleted file mode 100644
index 406e7f3..0000000
--- a/tests/example-empty-file/expected_results.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "version": 1,
-  "status": "fail",
-  "message": "Starting Vader: 1 suite(s), 5 case(s)\n  Starting Vader: /solution/example_empty_file.vader\n    (1/5) [EXECUTE] year not divisible by 4: common year\n    (1/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (2/5) [EXECUTE] year divisible by 4, not divisible by 100: leap year\n    (2/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (3/5) [EXECUTE] year divisible by 100, not divisible by 400: common year\n    (3/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (4/5) [EXECUTE] year divisible by 400: leap year\n    (4/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (5/5) [EXECUTE] year divisible by 200, not divisible by 400: common year\n    (5/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n  Success/Total: 0/5\nSuccess/Total: 0/5 (assertions: 0/0)\n\n\u001b[?1049h\u001b[22;0;0t\u001b[>4;2m\u001b[?1h\u001b=\u001b[?2004h\u001b[?1004h\u001b[24;1H\u001b[?2004l\u001b[>4;m\u001b[?2004h\u001b[>4;2m\u001b[?2004l\u001b[>4;m\u001b[23;2t\u001b[23;1t\u001b[?1004l\u001b[?2004l\u001b[?1l\u001b>\u001b[?1049l\u001b[23;0;0t\u001b[>4;m"
-}
diff --git a/tests/example-partial-fail/example_partial_fail.vader b/tests/example-partial-fail/example_partial_fail.vader
deleted file mode 100644
index ebc8488..0000000
--- a/tests/example-partial-fail/example_partial_fail.vader
+++ /dev/null
@@ -1,28 +0,0 @@
-"
-" Version: 1.4.0
-"
-
-Execute (year not divisible by 4: common year):
-  let year = 2015
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 4, not divisible by 100: leap year):
-  let year = 1996
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 100, not divisible by 400: common year):
-  let year = 2100
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 400: leap year):
-  let year = 2000
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 200, not divisible by 400: common year):
-  let year = 1800
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
diff --git a/tests/example-partial-fail/example_partial_fail.vim b/tests/example-partial-fail/example_partial_fail.vim
deleted file mode 100644
index 84b2ea0..0000000
--- a/tests/example-partial-fail/example_partial_fail.vim
+++ /dev/null
@@ -1,3 +0,0 @@
-function! LeapYear(year) abort
-  return a:year % 401 == 0 || a:year % 4 == 0 && a:year % 100 != 0
-endfunction
diff --git a/tests/example-partial-fail/expected_results.json b/tests/example-partial-fail/expected_results.json
deleted file mode 100644
index 0abdd1e..0000000
--- a/tests/example-partial-fail/expected_results.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "version": 1,
-  "status": "fail",
-  "message": "Starting Vader: 1 suite(s), 5 case(s)\n  Starting Vader: /solution/example_partial_fail.vader\n    (1/5) [EXECUTE] year not divisible by 4: common year\n    (2/5) [EXECUTE] year divisible by 4, not divisible by 100: leap year\n    (3/5) [EXECUTE] year divisible by 100, not divisible by 400: common year\n    (4/5) [EXECUTE] year divisible by 400: leap year\n    (4/5) [EXECUTE] (X) 0 should be equal to 1\n    (5/5) [EXECUTE] year divisible by 200, not divisible by 400: common year\n  Success/Total: 4/5\nSuccess/Total: 4/5 (assertions: 4/5)\n\n\u001b[?1049h\u001b[22;0;0t\u001b[>4;2m\u001b[?1h\u001b=\u001b[?2004h\u001b[?1004h\u001b[24;1H\u001b[?2004l\u001b[>4;m\u001b[?2004h\u001b[>4;2m\u001b[?2004l\u001b[>4;m\u001b[23;2t\u001b[23;1t\u001b[?1004l\u001b[?2004l\u001b[?1l\u001b>\u001b[?1049l\u001b[23;0;0t\u001b[>4;m"
-}
diff --git a/tests/example-success/example_success.vader b/tests/example-success/example_success.vader
deleted file mode 100644
index ebc8488..0000000
--- a/tests/example-success/example_success.vader
+++ /dev/null
@@ -1,28 +0,0 @@
-"
-" Version: 1.4.0
-"
-
-Execute (year not divisible by 4: common year):
-  let year = 2015
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 4, not divisible by 100: leap year):
-  let year = 1996
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 100, not divisible by 400: common year):
-  let year = 2100
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 400: leap year):
-  let year = 2000
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 200, not divisible by 400: common year):
-  let year = 1800
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
diff --git a/tests/example-success/example_success.vim b/tests/example-success/example_success.vim
deleted file mode 100644
index 2c20284..0000000
--- a/tests/example-success/example_success.vim
+++ /dev/null
@@ -1,3 +0,0 @@
-function! LeapYear(year) abort
-  return a:year % 400 == 0 || a:year % 4 == 0 && a:year % 100 != 0
-endfunction
diff --git a/tests/example-success/expected_results.json b/tests/example-success/expected_results.json
deleted file mode 100644
index 6c2223e..0000000
--- a/tests/example-success/expected_results.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "version": 1,
-  "status": "pass"
-}
diff --git a/tests/example-syntax-error/example_syntax_error.vader b/tests/example-syntax-error/example_syntax_error.vader
deleted file mode 100644
index ebc8488..0000000
--- a/tests/example-syntax-error/example_syntax_error.vader
+++ /dev/null
@@ -1,28 +0,0 @@
-"
-" Version: 1.4.0
-"
-
-Execute (year not divisible by 4: common year):
-  let year = 2015
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 4, not divisible by 100: leap year):
-  let year = 1996
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 100, not divisible by 400: common year):
-  let year = 2100
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 400: leap year):
-  let year = 2000
-  let expected = 1
-  AssertEqual expected, LeapYear(year)
-
-Execute (year divisible by 200, not divisible by 400: common year):
-  let year = 1800
-  let expected = 0
-  AssertEqual expected, LeapYear(year)
diff --git a/tests/example-syntax-error/example_syntax_error.vim b/tests/example-syntax-error/example_syntax_error.vim
deleted file mode 100644
index 1ad4b8f..0000000
--- a/tests/example-syntax-error/example_syntax_error.vim
+++ /dev/null
@@ -1 +0,0 @@
-FUNCTIONNNNN @#@#!
\ No newline at end of file
diff --git a/tests/example-syntax-error/expected_results.json b/tests/example-syntax-error/expected_results.json
deleted file mode 100644
index 7390bd7..0000000
--- a/tests/example-syntax-error/expected_results.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "version": 1,
-  "status": "fail",
-  "message": "Starting Vader: 1 suite(s), 5 case(s)\n  Starting Vader: /solution/example_syntax_error.vader\n    (1/5) [EXECUTE] year not divisible by 4: common year\n    (1/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (2/5) [EXECUTE] year divisible by 4, not divisible by 100: leap year\n    (2/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (3/5) [EXECUTE] year divisible by 100, not divisible by 400: common year\n    (3/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (4/5) [EXECUTE] year divisible by 400: leap year\n    (4/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n    (5/5) [EXECUTE] year divisible by 200, not divisible by 400: common year\n    (5/5) [EXECUTE] (X) Vim(call):E117: Unknown function: LeapYear\n      > command line..function <SNR>26_vader[5]..vader#run[63]..<SNR>28_run[42]..<SNR>28_execute[2]..vader#window#execute[11]..script line 3\n  Success/Total: 0/5\nSuccess/Total: 0/5 (assertions: 0/0)\n\n\u001b[?1049h\u001b[22;0;0t\u001b[>4;2m\u001b[?1h\u001b=\u001b[?2004h\u001b[?1004h\u001b[24;1H\u001b[?2004l\u001b[>4;m\u001b[?2004h\u001b[>4;2m\u001b[?2004l\u001b[>4;m\u001b[23;2t\u001b[23;1t\r\n\u001b[?1004l\u001b[?2004l\u001b[?1l\u001b>\u001b[?1049l\u001b[23;0;0t\u001b[>4;m"
-}
diff --git a/tests/example-vim9script/example_vim9script.vader b/tests/example-vim9script/example_vim9script.vader
deleted file mode 100644
index cd87cef..0000000
--- a/tests/example-vim9script/example_vim9script.vader
+++ /dev/null
@@ -1,3 +0,0 @@
-Execute (returns true):
-  let expected = 2
-  AssertEqual expected, ReturnsTwo()
diff --git a/tests/example-vim9script/example_vim9script.vim b/tests/example-vim9script/example_vim9script.vim
deleted file mode 100644
index 4620fe3..0000000
--- a/tests/example-vim9script/example_vim9script.vim
+++ /dev/null
@@ -1,7 +0,0 @@
-vim9script
-
-# this is a vim9script comment
-
-def g:ReturnsTwo(): number
-    return 2
-enddef
diff --git a/tests/example-vim9script/expected_results.json b/tests/example-vim9script/expected_results.json
deleted file mode 100644
index 6c2223e..0000000
--- a/tests/example-vim9script/expected_results.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "version": 1,
-  "status": "pass"
-}

From f751d3f81c5e3a50692aa31996a8d89821e7d0ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 17:27:27 -0700
Subject: [PATCH 06/10] Remove sed normalization

---
 bin/run-tests.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 59e6f94..0dcf940 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -22,13 +22,6 @@ for test_dir in tests/*; do
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
-    # Normalize the results file
-    sed -i -E \
-        -e 's/Elapsed time: [0-9]+\.[0-9]+ sec\.//' \
-        -e 's~/tmp/[^/]+/[^,]+,\s*~~g' \
-        -e "s~${test_dir_path}~/solution~g" \
-        "${results_file_path}"
-
     echo "${test_dir_name}: comparing results.json to expected_results.json"
     diff "${results_file_path}" "${expected_results_file_path}"
 

From abd087147225a9405d4f1d4eccd6165747815438 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 17:45:26 -0700
Subject: [PATCH 07/10] Add v2 test runner

---
 .vimrc           |   4 --
 bin/run-tests.sh |   9 +++-
 bin/run.sh       |  27 +++--------
 lib/runner.vim   | 120 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 134 insertions(+), 26 deletions(-)
 delete mode 100644 .vimrc
 create mode 100644 lib/runner.vim

diff --git a/.vimrc b/.vimrc
deleted file mode 100644
index 928ab8a..0000000
--- a/.vimrc
+++ /dev/null
@@ -1,4 +0,0 @@
-filetype off
-set runtimepath+=/opt/test-runner/vader.vim
-filetype plugin on
-syntax enable
diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 0dcf940..e911091 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -19,15 +19,22 @@ for test_dir in tests/*; do
     test_dir_path=$(realpath "${test_dir}")
     results_file_path="${test_dir_path}/results.json"
     expected_results_file_path="${test_dir_path}/expected_results.json"
+    formatted_results_file_path="${results_file_path}.formatted"
+    formatted_expected_results_file_path="${expected_results_file_path}.formatted"
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
+    jq -S . "${results_file_path}" > "${formatted_results_file_path}"
+    jq -S . "${expected_results_file_path}" > "${formatted_expected_results_file_path}"
+
     echo "${test_dir_name}: comparing results.json to expected_results.json"
-    diff "${results_file_path}" "${expected_results_file_path}"
+    diff "${formatted_results_file_path}" "${formatted_expected_results_file_path}"
 
     if [ $? -ne 0 ]; then
         exit_code=1
     fi
+
+    rm -f "${formatted_results_file_path}" "${formatted_expected_results_file_path}"
 done
 
 exit ${exit_code}
diff --git a/bin/run.sh b/bin/run.sh
index e3f5bb3..df69404 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -15,7 +15,7 @@
 # Example:
 # ./bin/run.sh two-fer /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/
 
-# If any required arguments is missing, print the usage and exit
+# If any required argument is missing, print the usage and exit.
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     echo "usage: ./bin/run.sh exercise-slug /absolute/path/to/two-fer/solution/folder/ /absolute/path/to/output/directory/"
     exit 1
@@ -25,33 +25,18 @@ slug="$1"
 input_dir="${2%/}"
 output_dir="${3%/}"
 exercise="${slug//-/_}"
-export implementation_file="${input_dir}/${exercise}.vim"
+implementation_file="${input_dir}/${exercise}.vim"
 tests_file="${input_dir}/${exercise}.vader"
 results_file="${output_dir}/results.json"
 
-# Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
+vim -Nu NONE -i NONE -n -es -S lib/runner.vim -- \
+    "${implementation_file}" "${tests_file}" "${results_file}" "${input_dir}" /opt/vader.vim
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(TERM=xterm HOME=/dev/null vim -XNu .vimrc -i NONE -c ":so $implementation_file" -c "Vader! ${tests_file}" 2>&1)
-
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
-if [ $? -eq 0 ] && [ "${test_output}" != "[EXECUTE] (X)" ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
-    # Strip off the logging information
-    sanitized_test_output=$(printf "${test_output}" | sed -n '/Starting Vader/,$p')
-
-    # Manually add colors to the output to help scanning the output for errors
-    colorized_test_output=$(echo "${sanitized_test_output}" \
-         | GREP_COLOR='01;31' grep --color=always -E -e '.*\[EXECUTE\] \(X\).*|$' \
-         | GREP_COLOR='01;32' grep --color=always -E -e '.*\[EXECUTE\].*|$')
-
-    jq -n --arg output "${colorized_test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+if [ ! -s "${results_file}" ]; then
+    jq -n '{version: 2, status: "error", message: "The Vim test runner could not start. Please start a thread on the Exercism forums.", tests: []}' > "${results_file}"
 fi
 
 echo "${slug}: done"
diff --git a/lib/runner.vim b/lib/runner.vim
new file mode 100644
index 0000000..044f5b2
--- /dev/null
+++ b/lib/runner.vim
@@ -0,0 +1,120 @@
+" Orchestrates a Vader run and writes Exercism test-runner v2 JSON.
+set nocompatible
+
+let s:arguments = v:argv[index(v:argv, '--') + 1 :]
+let s:solution_file = s:arguments[0]
+let s:test_file = s:arguments[1]
+let s:results_file = s:arguments[2]
+let s:input_dir = s:arguments[3]
+let s:vader_path = s:arguments[4]
+
+function! s:run() abort
+  try
+    if !filereadable(s:solution_file)
+      call s:error('Solution file not found: '.s:solution_file)
+      return
+    endif
+    if !filereadable(s:test_file)
+      call s:error('Test file not found: '.s:test_file)
+      return
+    endif
+    if !isdirectory(s:vader_path)
+      call s:error('Vader test framework is not installed.')
+      return
+    endif
+
+    execute 'set runtimepath^='.fnameescape(s:vader_path)
+    runtime plugin/vader.vim
+
+    execute 'source '.fnameescape(s:solution_file)
+    try
+      let cases = vader#parser#parse(s:test_file, 1, 0)
+    catch
+      call s:error('Vader test file syntax error.')
+      return
+    endtry
+
+    execute 'silent Vader '.fnameescape(s:test_file)
+    let report = split(get(g:, 'vader_report', ''), "\n")
+    let errors_by_line = {}
+    for vader_error in get(g:, 'vader_errors', [])
+      let errors_by_line[string(vader_error.lnum)] = vader_error
+    endfor
+
+    let tests = []
+    let all_errors = 1
+    let failed = 0
+    for case in cases
+      let test = {
+            \ 'name': s:case_name(case),
+            \ 'status': 'pass',
+            \ 'test_code': join(case.execute, "\n"),
+            \ }
+      let vader_error = get(errors_by_line, string(case.lnum), {})
+      if !empty(vader_error)
+        let message = s:failure_message(vader_error, report)
+        let test.status = s:failure_status(message)
+        let test.message = s:clean_message(message)
+      endif
+      let all_errors = all_errors && test.status ==# 'error'
+      let failed = failed || test.status !=# 'pass'
+      call add(tests, test)
+    endfor
+
+    if empty(tests)
+      call s:error('No Vader test cases were executed.')
+    elseif all_errors
+      call s:error(get(tests[0], 'message', 'All Vader test cases errored.'))
+    else
+      call s:write({
+            \ 'version': 2,
+            \ 'status': failed ? 'fail' : 'pass',
+            \ 'message': v:null,
+            \ 'tests': tests,
+            \ })
+    endif
+  catch
+    call s:error(v:exception)
+  endtry
+endfunction
+
+function! s:write(document) abort
+  call writefile([json_encode(a:document)], s:results_file)
+endfunction
+
+function! s:clean_message(message) abort
+  if !empty(s:input_dir)
+    return substitute(a:message, '\V'.escape(s:input_dir, '\\').'\m', '<solution-dir>', 'g')
+  endif
+  return a:message
+endfunction
+
+function! s:error(message) abort
+  call s:write({
+        \ 'version': 2,
+        \ 'status': 'error',
+        \ 'message': s:clean_message(a:message),
+        \ 'tests': [],
+        \ })
+endfunction
+
+function! s:case_name(case) abort
+  let name = get(a:case.comment, 'execute', '')
+  return empty(name) ? printf('Test at line %d', a:case.lnum) : name
+endfunction
+
+function! s:failure_message(vader_error, report) abort
+  let report_line = matchstr(get(a:vader_error, 'text', ''), '(#\zs\d\+\ze)')
+  if !empty(report_line) && report_line >= 1 && report_line <= len(a:report)
+    let message = a:report[report_line - 1]
+    return substitute(message, '^\s*(\d\+/\d\+) \[[^]]*\] (X) ', '', '')
+  endif
+  return get(a:vader_error, 'text', 'Test failed.')
+endfunction
+
+function! s:failure_status(message) abort
+  return a:message =~# '^Vim\%(([^)]*)\)\?:' ? 'error' : 'fail'
+endfunction
+
+call s:run()
+qa!

From 5023610ae02db262f7235357b320aa15cf4f06f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 19:17:28 -0700
Subject: [PATCH 08/10] Use a tmpdir

---
 bin/run-tests.sh | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index e911091..7836669 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -12,15 +12,16 @@
 # ./bin/run-tests.sh
 
 exit_code=0
+temp_dir=$(mktemp -d) || exit 1
+trap 'rm -rf "${temp_dir}"' EXIT
 
-# Iterate over all test directories
-for test_dir in tests/*; do
-    test_dir_name=$(basename "${test_dir}")
-    test_dir_path=$(realpath "${test_dir}")
+run_test() {
+    test_dir_name=$(basename "${1}")
+    test_dir_path=$(realpath "${1}")
     results_file_path="${test_dir_path}/results.json"
     expected_results_file_path="${test_dir_path}/expected_results.json"
-    formatted_results_file_path="${results_file_path}.formatted"
-    formatted_expected_results_file_path="${expected_results_file_path}.formatted"
+    formatted_results_file_path="${temp_dir}/${test_dir_name}-results.json"
+    formatted_expected_results_file_path="${temp_dir}/${test_dir_name}-expected-results.json"
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
@@ -29,12 +30,14 @@ for test_dir in tests/*; do
 
     echo "${test_dir_name}: comparing results.json to expected_results.json"
     diff "${formatted_results_file_path}" "${formatted_expected_results_file_path}"
+}
+
+for test_dir in tests/*; do
+    run_test "${test_dir}"
 
     if [ $? -ne 0 ]; then
         exit_code=1
     fi
-
-    rm -f "${formatted_results_file_path}" "${formatted_expected_results_file_path}"
 done
 
 exit ${exit_code}

From 9868d2063096d5cc250c9a2dfeb7a8b5fb9742c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 19:25:47 -0700
Subject: [PATCH 09/10] Simplify Dockerfile

---
 .dockerignore | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index ca330b6..bd6832c 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,5 +1,3 @@
 *
 !bin/
-!bin/**
 !lib/
-!lib/**

From d2763157cfff6aa5bb682b8879ef0c886250b8b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A1s=20B=20Nagy?=
 <20251272+BNAndras@users.noreply.github.com>
Date: Mon, 5 Oct 2026 21:11:17 -0700
Subject: [PATCH 10/10] Use git clone instead of git init and fetch

---
 Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 89f38c5..9a7c345 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,10 +3,7 @@ FROM alpine:3.23.5@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952
 ARG VADER_REV=429b669e6158be3a9fc110799607c232e6ed8e29
 
 RUN apk add --no-cache bash git jq vim \
-    && git init /opt/vader.vim \
-    && git -C /opt/vader.vim remote add origin https://github.com/junegunn/vader.vim.git \
-    && git -C /opt/vader.vim fetch --depth 1 origin "$VADER_REV" \
-    && git -C /opt/vader.vim checkout --detach FETCH_HEAD \
+    && git clone --revision="$VADER_REV" --depth=1 https://github.com/junegunn/vader.vim.git /opt/vader.vim \
     && rm -rf /opt/vader.vim/.git
 
 COPY . /opt/test-runner