From 83e884d872b93fb0c136177e98a2dbd212acd5a2 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Fri, 13 Nov 2026 15:23:45 +0900 Subject: [PATCH] 0.1.2: the runner is dirun; the specification by its minor version tools/vm-runner runs the program by dirun with the image's profile (beside the image, as freebsd-disk-image installs it, else tools/freebsd.toml), passing OPENKAL_* and KAL_*; a test runner's events come back from the machine, and the run is limited to OKF_VM_TIMEOUT seconds (1800). tools/vm-run.py, a second implementation of the same conversation, is removed. The manifest declares openkal = "^0.25.0" (openkal clause 8). --- CHANGELOG.md | 7 ++ README.md | 49 +++++++++----- mcpp.toml | 5 +- tools/freebsd.toml | 57 ++++++++++++++++ tools/vm-run.py | 162 --------------------------------------------- tools/vm-runner | 16 +++-- 6 files changed, 112 insertions(+), 184 deletions(-) create mode 100644 tools/freebsd.toml delete mode 100755 tools/vm-run.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d2e0a78..4ee1904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.1.2 + +- `tools/vm-runner` runs the program by dirun with the image's profile (beside the image, as + `freebsd-disk-image` installs it, else `tools/freebsd.toml`); `tools/vm-run.py` is removed. +- The specification is declared as `^0.25.0` (openkal clause 8): a consumer that names a + later patch release resolves with this implementation. + ## 0.1.1 - `tools/vm-run.py` boots each run from a temporary overlay of the image diff --git a/README.md b/README.md index e97fdfb..b4ad771 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,24 @@ written on the kernel's own system-call interface. ```toml [target.'cfg(os = "freebsd")'.dependencies] -openkal-freebsd = "0.1.1" +openkal-freebsd = "0.1.2" ``` +## What 0.1.2 changes + +**The runner is dirun.** `tools/vm-runner` boots the machine by dirun, the one +program that runs a command inside a booted disk image, with the image's profile +(`dirun.toml` beside the image, as the package `freebsd-disk-image` installs it, +else `tools/freebsd.toml`); `tools/vm-run.py`, a second implementation of the same +conversation, is removed. What it did is dirun's: the overlay, the program +delivered on a read-only FAT disk and run from its copy, `OPENKAL_*` and `KAL_*` +passed, and a test runner's events read back from the machine. + +**The specification by its minor version.** The manifest declares `openkal = +"^0.25.0"`: an implementation written against a patch release conforms to every +later patch of that minor version (openkal clause 8), and a consumer that names +openkal 0.25.1 resolves with it. + ## What it is FreeBSD's kernel interface is stable: a program built for FreeBSD 14 runs on @@ -105,12 +120,11 @@ runs above openkal-musl on the `*-freebsd-musl` rows. - `tools/defer-runner` puts the suite and the variables it is run with in a directory, where CI runs it inside a FreeBSD VM. -- `tools/vm-run.py` runs a program in a local FreeBSD VM image for development - (`OKF_VM_IMAGE`, `tools/vm-runner`). It forwards `OPENKAL_*`, `KAL_*` and the - mtest protocol's variables, so the suite can report through mtest once the - runner's listing pass honours `--run-under` (mtest 0.6.0). Each run boots a - temporary overlay of the image, so the image is never written to and two runs - at once do not meet. +- `tools/vm-runner` runs a program in a local FreeBSD VM image for development + (`OKF_VM_IMAGE`), by dirun. It passes `OPENKAL_*` and `KAL_*`, and a test + runner's events come back from the machine. Each run boots a temporary overlay + of the image, so the image is never written to and two runs at once do not + meet. ### The runner declaration @@ -123,15 +137,18 @@ runner = ["/path/to/openkal-freebsd/tools/vm-runner", "{}"] ``` The declaration's `{}` is the built artefact. `vm-runner` boots the FreeBSD -image that `OKF_VM_IMAGE` names under `qemu-system-x86_64` (KVM where the host -has it, the software machine otherwise), offers the artefact on a read-only -drive, runs it as root on the serial console, writes what it writes to the -standard output and error, answers with its exit status, and shuts the machine -down; the temporary overlay it booted from is discarded. The disk is the -FreeBSD 14.5-RELEASE boot disk of the `xim` package `freebsd-disk` (The -FreeBSD Project's release, installed from the official installer disc; -published separately), and `qemu-system-x86_64` with `qemu-img` are the -host's dependencies. +image that `OKF_VM_IMAGE` names by `dirun` (under `qemu-system-x86_64`, KVM +where the host has it, the software machine otherwise), delivers the artefact +on a read-only FAT disk, runs it as root on the serial console, writes what it +writes to the standard output, and answers with its exit status; the temporary +overlay it booted from is discarded. The disk is the FreeBSD 14.5-RELEASE boot +disk of the `xim` package `freebsd-disk-image` (The FreeBSD Project's release, +installed from the official installer disc), which installs the profile beside +it; `dirun`, `qemu-system-x86_64` and `qemu-img` are the host's dependencies. +Without a declaration, mcpp states the same runner for the FreeBSD rows of an +x86_64 Linux or macOS host itself: `dirun --profile +{pkg:freebsd-disk-image}/dirun.toml -- {}`, its packages installed by the first +run. The targets need mcpp 2026.11.10.1 or later. diff --git a/mcpp.toml b/mcpp.toml index d7cad68..f7cebb8 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-freebsd" -version = "0.1.1" +version = "0.1.2" description = "An implementation of openkal for FreeBSD, written on the kernel's own system-call interface." license = "Apache-2.0" authors = ["mcpplibs"] @@ -36,7 +36,8 @@ provides-interfaces = [ ] [dependencies] -openkal = "0.25.0" +# Every patch release of the specification's minor version (openkal clause 8). +openkal = "^0.25.0" [build] sources = ["src/*.cpp"] diff --git a/tools/freebsd.toml b/tools/freebsd.toml new file mode 100644 index 0000000..72f4bdd --- /dev/null +++ b/tools/freebsd.toml @@ -0,0 +1,57 @@ +# How tools/vm-runner boots a FreeBSD 14.5-RELEASE x86_64 image by dirun: the loader's menu is left +# for its prompt, the console moved to the serial line, root logs in, the files disk is mounted at +# /mnt/dirun, and a test runner's events come back. It is the profile the package +# freebsd-disk-image ships beside its image, without the image's name: the image is OKF_VM_IMAGE. +name = "freebsd-14.5-x86_64" + +[machine] +arch = "x86_64" +memory = 1024 +smp = 2 + +[[boot]] +expect = "Autoboot in" +send = " 3" # any key stops the countdown; 3 leaves the menu for the loader's prompt +line = false +pace = 300 # the loader polls its console: faster input is dropped +timeout = 120 + +[[boot]] +expect = "OK " +send = "set console=comconsole" +pace = 50 + +[[boot]] +expect = "OK " +send = "boot" +pace = 50 + +[[boot]] +expect = "login: " +send = "root" +timeout = 300 + +[[boot]] +expect = "Password:" +send = "root" +optional = true + +[[boot]] +expect = "# |\\$ " + +[command] +prepare = ["stty -echo"] +line = "echo @@BEGIN@@; {env}{argv} < /dev/null; echo @@STATUS=$?@@" +env = "env {assignments} " +quote = "posix" +begin = "@@BEGIN@@" +status = "@@STATUS=(\\d+)@@" +line-max = 200 # a longer line is written to the files disk and run from there +script = "sh {script}" + +[files] +mount = ["mkdir -p /mnt/dirun", "mount -r -t msdosfs /dev/vtbd1s1 /mnt/dirun"] +at = "/mnt/dirun" + +[relay] +report = "/tmp/dirun-mtest.ndjson" diff --git a/tools/vm-run.py b/tools/vm-run.py deleted file mode 100755 index 2ef5df6..0000000 --- a/tools/vm-run.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -"""Runs a program in a FreeBSD virtual machine and answers as the program did. - - vm-run.py [--image ] [--file ]... -- [arguments...] - -The image is , or OKF_VM_IMAGE where --image is absent. The machine is -booted under qemu from a temporary overlay of (qemu-img; nothing is -written to itself, and two runs at once do not meet), the files are -offered to it on a read-only FAT drive and copied to /tmp/run, and the program -is run there as root on the serial console. What it writes is written here, and -its status is this program's status. Every argument is passed through single -quotes, so the command line is the one given. KVM is used where the host has it -(a writable /dev/kvm) and the software machine otherwise. - -THE PROGRAM'S ENVIRONMENT. Variables named OPENKAL_* and KAL_* are passed to -the program. Where a test runner speaks the mtest protocol to it (MTEST_PROTOCOL -and MTEST_REPORT set), the program writes its events into a file of the machine, -which is appended to MTEST_REPORT here when it ends; MTEST_MODE and MTEST_SELECT -(a file, offered to the machine) are passed as well. - -It is how this repository's conformance suite is run on FreeBSD from a Linux -host: the suite's runner (OPENKAL_CONFORMANCE_RUNNER), which a consumer states -as a runner declaration naming tools/vm-runner. The image's kernel console is -moved to the serial line at the loader prompt, as the official image boots to -the video console. -""" -import argparse, os, pty, re, select, shutil, subprocess, sys, tempfile, time - -ap = argparse.ArgumentParser() -ap.add_argument("--image", default=os.environ.get("OKF_VM_IMAGE")) -ap.add_argument("--file", action="append", default=[]) -ap.add_argument("--memory", default="2048") -ap.add_argument("--timeout", type=int, default=1800) -ap.add_argument("command", nargs=argparse.REMAINDER) -args = ap.parse_args() -command = args.command[1:] if args.command[:1] == ["--"] else args.command -if not command: - ap.error("no program to run") -if not args.image: - ap.error("no image: --image or OKF_VM_IMAGE") - -share = tempfile.mkdtemp(prefix="vm-run-") -overlay = tempfile.mkdtemp(prefix="vm-run-overlay-") -forward = {k: v for k, v in os.environ.items() if k.startswith("OPENKAL_") or k.startswith("KAL_")} -report = os.environ.get("MTEST_REPORT") if os.environ.get("MTEST_PROTOCOL") else None -if report: - forward["MTEST_PROTOCOL"] = os.environ["MTEST_PROTOCOL"] - forward["MTEST_REPORT"] = "/tmp/run/vm-run-report.ndjson" - if os.environ.get("MTEST_MODE"): - forward["MTEST_MODE"] = os.environ["MTEST_MODE"] - if os.environ.get("MTEST_SELECT") and os.path.isfile(os.environ["MTEST_SELECT"]): - shutil.copy2(os.environ["MTEST_SELECT"], os.path.join(share, "vm-run-select.txt")) - forward["MTEST_SELECT"] = "/tmp/run/vm-run-select.txt" -for f in args.file + [command[0]]: - if os.path.isfile(f): - shutil.copy2(f, os.path.join(share, os.path.basename(f))) -# A program offered from here is run from the copy; one named otherwise is the -# machine's own (`/usr/bin/lldb'). -program = "./" + os.path.basename(command[0]) if os.path.isfile(command[0]) else command[0] - -def quote(a): - return "'" + a.replace("'", "'\\''") + "'" - -line = " ".join(["env"] + [quote(f"{k}={v}") for k, v in sorted(forward.items())] - + [quote(program)] + [quote(a) for a in command[1:]]) -overlay_disk = os.path.join(overlay, "run.qcow2") -subprocess.run(["qemu-img", "create", "-f", "qcow2", "-b", os.path.abspath(args.image), - "-F", "qcow2", overlay_disk], check=True, - stdout=subprocess.DEVNULL) -argv = ["qemu-system-x86_64", "-m", args.memory, "-smp", "2", "-nographic", - "-drive", f"file={overlay_disk},if=virtio,format=qcow2", - "-drive", f"file=fat:{share},if=virtio,format=raw,readonly=on", - "-nic", "none"] -if os.access("/dev/kvm", os.W_OK): - argv.insert(1, "-enable-kvm") -pid, fd = pty.fork() -if pid == 0: - os.execvp(argv[0], argv) - -buf = b"" -def read_until(pattern, timeout): - global buf - end = time.time() + timeout - rx = re.compile(pattern) - while time.time() < end: - r, _, _ = select.select([fd], [], [], 0.2) - if r: - try: - chunk = os.read(fd, 65536) - except OSError: - return None - buf += chunk - m = rx.search(buf) - if m: - return m - return None - -def send(text): - for ch in text: - os.write(fd, ch.encode()) - time.sleep(0.02) - -def stop(status): - try: - os.kill(pid, 9) - except OSError: - pass - shutil.rmtree(share, ignore_errors=True) - shutil.rmtree(overlay, ignore_errors=True) - sys.exit(status) - -if read_until(rb"Autoboot in", 180): - send(" "); time.sleep(0.5); send("3") - read_until(rb"OK ", 30); time.sleep(1); buf = b"" - send("set console=comconsole\r"); read_until(rb"OK ", 10); time.sleep(1); buf = b"" - send("boot\r") -if not read_until(rb"login: ", 300): - send("\r") - if not read_until(rb"login: ", 60): - sys.stderr.write("vm-run: the machine gave no login prompt\n") - stop(125) -buf = b"" -send("root\r") -read_until(rb"(# |Password:)", 30) -if b"Password:" in buf: - send("\r"); read_until(rb"# ", 30) - -# The program's output between two marks, and its status after the second. -buf = b"" -send("sh\r"); read_until(rb"\$ |# ", 10) -buf = b"" -send("mkdir -p /mnt/s /tmp/run && mount -t msdosfs -o ro /dev/vtbd1s1 /mnt/s && cp /mnt/s/* /tmp/run/ " - "&& chmod +x /tmp/run/* && cd /tmp/run && stty -echo; echo @@VM-RUN-BEGIN@@; " - + line + "; rc=$?; echo; echo @@VM-RUN-STATUS=$rc@@\r") -m = read_until(rb"@@VM-RUN-STATUS=(\d+)@@", args.timeout) -if not m: - sys.stderr.write("vm-run: the program did not finish within the time allowed\n") - stop(124) -text = buf.decode(errors="replace").replace("\r", "") -begin = text.rfind("@@VM-RUN-BEGIN@@\n") -body = text[begin + len("@@VM-RUN-BEGIN@@\n"):text.rfind("\n@@VM-RUN-STATUS=")] if begin >= 0 else text -# `echo' after the program added one line feed, so that the mark begins a line. -if body.endswith("\n"): - body = body[:-1] -sys.stdout.write(body + ("\n" if body and not body.endswith("\n") else "")) -sys.stdout.flush() -status = int(m.group(1)) -if report: - buf = b"" - send("echo @@VM-RUN-REPORT@@; cat /tmp/run/vm-run-report.ndjson 2>/dev/null; echo; echo @@VM-RUN-REPORT-END@@\r") - if read_until(rb"@@VM-RUN-REPORT-END@@\r?\n", 120): - text = buf.decode(errors="replace").replace("\r", "") - a = text.find("@@VM-RUN-REPORT@@\n") - b = text.rfind("@@VM-RUN-REPORT-END@@") - if a >= 0 and b > a: - events = [l for l in text[a + len("@@VM-RUN-REPORT@@\n"):b].split("\n") if l.startswith("{")] - with open(report, "a") as out: - for l in events: - out.write(l + "\n") -send("shutdown -p now\r") -read_until(rb"Uptime:", 60) -stop(status) diff --git a/tools/vm-runner b/tools/vm-runner index 7493130..2ec0df7 100755 --- a/tools/vm-runner +++ b/tools/vm-runner @@ -1,5 +1,13 @@ #!/bin/sh -# The runner a runner declaration names on a Linux host: the program, run in -# the FreeBSD virtual machine whose disk OKF_VM_IMAGE names (tools/vm-run.py). -# The declaration's payload is the first argument, as with any command. -exec python3 "$(dirname "$0")/vm-run.py" --image "${OKF_VM_IMAGE:?the FreeBSD disk image}" -- "$@" +# The runner a runner declaration names on a Linux or macOS host: the program, run in a FreeBSD +# virtual machine by dirun. The machine boots the image OKF_VM_IMAGE names (a temporary overlay of +# it; the image is never written), with the profile beside it (dirun.toml, as the package +# freebsd-disk-image ships it) or else tools/freebsd.toml. The program is delivered and run from its +# copy; OPENKAL_* and KAL_* are passed to it, and a test runner's events come back. The run is +# limited to OKF_VM_TIMEOUT seconds (1800). +here=$(dirname "$0") +image=${OKF_VM_IMAGE:?the FreeBSD disk image (the package freebsd-disk-image installs one)} +profile=$(dirname "$image")/dirun.toml +[ -f "$profile" ] || profile=$here/freebsd.toml +exec dirun --image "$image" --profile "$profile" --timeout "${OKF_VM_TIMEOUT:-1800}" \ + --pass 'OPENKAL_*' --pass 'KAL_*' -- "$@"