Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,35 @@ Notable changes to shen-lua. Release notes for earlier versions are under

### Fixed

- The native typechecker no longer returns wrong verdicts on a kernel whose
type-checker defuns differ from the ones it was written against. On
ShenOSKernel 43.1 with the typecheck deref hoist (shen-sources
`perf/typecheck-deref-hoist`), the translated `shen.by-hypothesis` called
the new `shen.by-hypothesis-h`, which had no native definition. The call
raised a plain Lua error. That error reached callers as an ordinary Shen
exception, which `trap-error` (and the Lua `shen.typecheck` API, which
fails closed) read as "does not typecheck". Every typecheck failed and
the process exited 0. Now:
- The translated drivers are the closure of the t-star roots under
"calls by name": any `t-star.kl` defun a driver calls is translated too,
so the hoisted kernel's `shen.by-hypothesis-h` runs natively.
- Before the first typecheck, every kernel defun the port reimplements
instead of translating is fingerprinted against the versions it
implements. That covers the hand-ported `shen.typecheck`,
`lookupsig`/`sigf`, `search-user-datatypes`(`-h`), `show` and
`maxinfexceeded?`, plus the `prolog.kl` primitives and builtin goals.
The fingerprint is gensym-insensitive; regenerate it with
`scripts/kernel-fingerprints.lua`. The set also checks that every driver
call resolves.
- A mismatch makes typechecks use the kernel's own KL typechecker, and
stderr names the defun once. A predicate that is missing at run time
raises a distinct sentinel, and the query is rerun on the KL engine.
- `SHEN_TYPECHECK_NATIVE=require` raises an error instead of falling back.
- `test/kernel_conformance_spec.lua` runs the native typechecker on the
vendored 43.1 and the deref-hoisted 43.1 (fixture under
`test/fixtures/`) against the kernel's KL typechecker. It also runs it
on deliberately mismatched kernels.

- Deep non-tail recursion no longer overflows the Lua stack, in user code
or in the kernel's own tree walkers. `(depth 12000)` for a plain
`(+ 1 (depth (- N 1)))` recursion failed with `stack overflow` (it reached
Expand Down
38 changes: 24 additions & 14 deletions prolog_compile.lua
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,16 @@ local BUILTIN_GOALS = {
["call"] = true,
}

-- A call to another predicate by name. Records the callee (the driver
-- closure in typecheck_native.lua follows these) and resolves through MISS so
-- a name with no native predicate raises E.native_miss rather than calling
-- nil.
local function np_ref(ctx, nm)
ctx.callees[nm] = true
local q = string.format("%q", nm)
return "(NP[" .. q .. "] or MISS(" .. q .. "))"
end

local function refuse(ctx, why)
ctx.failed = why
error(ctx.fail, 0)
Expand Down Expand Up @@ -742,8 +752,8 @@ compile_value = function(ctx, e, env, out, d)
for _, l in ipairs(gb) do out[#out + 1] = l end
local gname = sname(g[1])
if BUILTIN_GOALS[gname] then refuse(ctx, "builtin inside fork") end
out[#out + 1] = indent(d) .. guard .. r .. " = NP[" ..
string.format("%q", gname) .. "](" ..
out[#out + 1] = indent(d) .. guard .. r .. " = " ..
np_ref(ctx, gname) .. "(" ..
table.concat(parts, ", ") .. (gn > 0 and ", " or "") ..
nk .. ", " .. kv .. ")" .. (first and "" or " end")
first = false
Expand Down Expand Up @@ -774,7 +784,7 @@ compile_value = function(ctx, e, env, out, d)
local k = compile_cont(ctx, a[an], env, out, d)
parts[#parts + 1] = n
parts[#parts + 1] = k
return "NP[" .. string.format("%q", nm) .. "](" .. table.concat(parts, ", ") .. ")"
return np_ref(ctx, nm) .. "(" .. table.concat(parts, ", ") .. ")"
end

refuse(ctx, "unsupported form")
Expand Down Expand Up @@ -1002,6 +1012,7 @@ local CONS, CAR, CDR, LZD = E.cons, E.car, E.cdr, E.lazyderef
local NEWVAR, POPVAR = E.newvar, E.popvar
local UNIFY, UNIFYOC, BIND1 = E.unify, E.unify_oc, E.bind1
local EQTERM = E.eqterm
local MISS = E.native_miss
local CUT, UNLOCK, LOCKOPEN = E.cut, E.unlock, E.lock_is_open
local INCINFS, THAW, THAWN, CAPREF = E.incinfs, E.thawH, E.thawN, E.capref
local GVAR, GCALL, GFINDALL = E.g_var, E.g_call, E.g_findall
Expand Down Expand Up @@ -1030,7 +1041,7 @@ local function translate_once(name, allparams, body, mode)
allparams[np - 1], allparams[np]

local ctx = setmetatable({
buf = {}, nlift = 0, nlocal = 0, fail = {}, liftnames = {},
buf = {}, nlift = 0, nlocal = 0, fail = {}, liftnames = {}, callees = {},
fnloc = nparams + 2, split = mode and true or false, wide = mode == "wide",
}, CtxMT)

Expand Down Expand Up @@ -1081,7 +1092,7 @@ local function translate_once(name, allparams, body, mode)
return nil, "luagen: " .. tostring(err) .. "\n" .. source, nil, err
end
local fn = chunk(E, NP, F, E.materialize, E.import_cached)
return fn, nil, source
return fn, nil, source, nil, ctx.callees
end

-- Let-chain splitting (see compile_value) costs a call per split point, so
Expand All @@ -1097,15 +1108,16 @@ local function width_error(lerr)
or lerr:find("too many lifted functions", 1, true) ~= nil
end

-- Returns fn, err, src, callees (the set of predicate names fn calls).
local function translate_core(name, allparams, body)
local fn, err, src, lerr = translate_once(name, allparams, body, false)
local fn, err, src, lerr, callees = translate_once(name, allparams, body, false)
if lerr and tostring(lerr):find("local variables", 1, true) then
fn, err, src, lerr = translate_once(name, allparams, body, "split")
fn, err, src, lerr, callees = translate_once(name, allparams, body, "split")
end
if width_error(lerr) then
fn, err, src = translate_once(name, allparams, body, "wide")
fn, err, src, lerr, callees = translate_once(name, allparams, body, "wide")
end
return fn, err, src
return fn, err, src, callees
end
M.translate_core = translate_core

Expand All @@ -1131,8 +1143,8 @@ function M.translate_defun(defunForm)
local name = sname(parts[2])
local allparams = lst2tbl(parts[3])
local body = parts[4]
local fn, err, src = translate_core(name, allparams, body)
return fn, err, src, name
local fn, err, src, callees = translate_core(name, allparams, body)
return fn, err, src, name, callees
end

-- ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1268,9 +1280,7 @@ local function install_dispatch_builtins()
error("native call: goal head is not a symbol")
end
local fn = NP[hv.name]
if not fn then
error("native call: no native predicate " .. hv.name)
end
if not fn then E.native_miss(hv.name) end
local args, an = {}, 0
local rest = E.lazyderef(E.cdr(d))
while rest >= E.CONS_BASE do
Expand Down
15 changes: 15 additions & 0 deletions prolog_engine.lua
Original file line number Diff line number Diff line change
Expand Up @@ -870,6 +870,21 @@ end
-- ---------------------------------------------------------------------------
M.NativePred = {}

-- A translated predicate calls another by name through NativePred. A name
-- with no native predicate used to be `nil(...)`: a plain Lua error that
-- trap-error turns into an ordinary Shen exception, so a typecheck that hit
-- it simply looked like a failed typecheck (wrong answer, exit 0). Call
-- sites now resolve through MISS, which raises this distinct sentinel; the
-- typecheck entry recognises it and reruns the whole query on the kernel's
-- KL engine (typecheck_native.lua).
local MissMT = { __tostring = function(m)
return "native prolog: no native predicate " .. tostring(m.name)
end }
function M.native_miss(name)
error(setmetatable({ name = name }, MissMT), 0)
end
function M.is_native_miss(x) return getmetatable(x) == MissMT end

-- ---------------------------------------------------------------------------
-- install: wiring into the running kernel. Inert until prolog_compile.lua
-- (Phase 2) and typecheck_native.lua (Phase 3) register their overrides.
Expand Down
47 changes: 47 additions & 0 deletions scripts/kernel-fingerprints.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
-- scripts/kernel-fingerprints.lua — regenerate typecheck_native.lua's KNOWN
-- table: the fingerprints of the kernel defuns the native typechecker
-- reimplements (typecheck_native.lua REIMPLEMENTED).
--
-- luajit scripts/kernel-fingerprints.lua LABEL KLDIR [LABEL KLDIR ...]
--
-- Prints the table body for every kernel given. Add a kernel ONLY after
-- checking that the native code implements each of its listed defuns: a
-- fingerprint in KNOWN is a claim that the port runs that defun's semantics.
local root = (arg[0]:gsub("scripts/[^/]*$", ""))
package.path = (root ~= "" and root or "./") .. "?.lua;" .. package.path
local TN = require("typecheck_native")

if #arg < 2 or #arg % 2 ~= 0 then
io.stderr:write("usage: kernel-fingerprints.lua LABEL KLDIR [LABEL KLDIR ...]\n")
os.exit(2)
end

local function slurp(path)
local f = assert(io.open(path, "r"))
local s = f:read("*a"); f:close()
return s
end

local seen = {} -- name -> { fp -> {labels} }, fp order
for i = 1, #arg, 2 do
local label, dir = arg[i], arg[i + 1]
local defsets = {}
for _, e in ipairs(TN.REIMPLEMENTED) do
local file = e[1]
defsets[file] = defsets[file] or TN.defuns_of(slurp(dir .. "/" .. file .. ".kl"))
local fp = TN.fingerprint_in(defsets[file], e[2])
local byname = seen[e[2]]
if not byname then byname = { order = {} }; seen[e[2]] = byname end
if not byname[fp] then byname[fp] = {}; byname.order[#byname.order + 1] = fp end
table.insert(byname[fp], label)
end
end

for _, e in ipairs(TN.REIMPLEMENTED) do
local byname = seen[e[2]]
local parts = {}
for _, fp in ipairs(byname.order) do
parts[#parts + 1] = string.format('["%s"] = "%s"', fp, table.concat(byname[fp], ", "))
end
io.write(string.format(' [%q] = { %s },\n', e[2], table.concat(parts, ", ")))
end
1 change: 1 addition & 0 deletions scripts/run-tests.lua
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ local specs = {
"test/interop_spec.lua",
"test/io_spec.lua",
"test/jit_spec.lua",
"test/kernel_conformance_spec.lua",
"test/library_spec.lua",
"test/numeric_literal_spec.lua",
"test/numeric_render_spec.lua",
Expand Down
17 changes: 17 additions & 0 deletions test/fixtures/kernel-43.1-deref-hoist/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# ShenOSKernel 43.1 with the typecheck deref hoist

`t-star.kl` is ShenOSKernel 43.1's `klambda/t-star.kl` with one kernel source
change applied: `perf(t-star): deref the checked term once, outside the
datatype and hypothesis loops` (shen-sources branch
`perf/typecheck-deref-hoist`, commit
`55964453e8a687ba2c9d107dbaebb23ef72996a9`, not yet upstream).

- SHA-256: `a0cedbb0334eb7f66fe5e2296056ea66d1e2f42c9efe176f6d1be2f0b28bf2ec`
- `shen.search-user-datatypes` and `shen.by-hypothesis` become plain defuns
that dereference their term arguments once and call the new Prolog
predicates `shen.search-user-datatypes-h` and `shen.by-hypothesis-h`.
- Every other defun is unchanged except for gensym renumbering.

`test/kernel_conformance_spec.lua` assembles a kernel directory from the
vendored `klambda/` with this file in place of `t-star.kl` and checks that the
native typechecker runs it (and agrees with the kernel's KL typechecker).
112 changes: 112 additions & 0 deletions test/fixtures/kernel-43.1-deref-hoist/t-star.kl

Large diffs are not rendered by default.

Loading
Loading