From 3a7cb4c655e25908c78d6499468376bfb2e4ee1f Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 12 Mar 2026 23:13:55 -0500 Subject: [PATCH 1/2] Replace scons with direct clang++ subprocess build Remove scons dependency and replace with build.py that compiles C++/Cython extensions using direct clang++ calls via subprocess. Supports parallel compilation and incremental builds via -MMD dependency tracking. Co-Authored-By: Claude Opus 4.6 --- .gitignore | 2 +- SConscript | 39 ------ SConstruct | 83 ------------ build.py | 227 ++++++++++++++++++++++++++++++++ msgq/.gitignore | 1 + msgq/__init__.py | 4 +- msgq/visionipc/__init__.py | 3 + pyproject.toml | 5 +- site_scons/site_tools/cython.py | 72 ---------- test.sh | 2 +- 10 files changed, 237 insertions(+), 201 deletions(-) delete mode 100644 SConscript delete mode 100644 SConstruct create mode 100644 build.py delete mode 100644 site_scons/site_tools/cython.py diff --git a/.gitignore b/.gitignore index 324effb5b..6ace24758 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,5 @@ test_runner libmessaging.* libmessaging_shared.* -.sconsign.dblite +*.d .mypy_cache/ \ No newline at end of file diff --git a/SConscript b/SConscript deleted file mode 100644 index c56ce994a..000000000 --- a/SConscript +++ /dev/null @@ -1,39 +0,0 @@ -Import('env', 'envCython', 'arch', 'common') - - -visionipc_dir = Dir('msgq/visionipc') - -# Build msgq -msgq_objects = env.SharedObject([ - 'msgq/ipc.cc', - 'msgq/event.cc', - 'msgq/impl_msgq.cc', - 'msgq/impl_fake.cc', - 'msgq/msgq.cc', -]) -msgq = env.Library('msgq', msgq_objects) -msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq, common]) - -# Build Vision IPC -vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc'] -if arch == "larch64": - vipc_files += ['visionbuf_ion.cc'] -else: - vipc_files += ['visionbuf.cc'] -vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files] - -vipc_objects = env.SharedObject(vipc_sources) -visionipc = env.Library('visionipc', vipc_objects) - - -vipc_libs = envCython["LIBS"] + [visionipc, msgq, common] -envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx', - LIBS=vipc_libs) - -if GetOption('extras'): - env.Program('msgq/test_runner', ['msgq/test_runner.cc', 'msgq/msgq_tests.cc'], LIBS=[msgq, common]) - env.Program(f'{visionipc_dir.abspath}/test_runner', - [f'{visionipc_dir.abspath}/test_runner.cc', f'{visionipc_dir.abspath}/visionipc_tests.cc'], - LIBS=['pthread'] + vipc_libs) - -Export('visionipc', 'msgq', 'msgq_python') diff --git a/SConstruct b/SConstruct deleted file mode 100644 index e38e79a12..000000000 --- a/SConstruct +++ /dev/null @@ -1,83 +0,0 @@ -import os -import platform -import subprocess -import sysconfig -import numpy as np - -arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() -if platform.system() == "Darwin": - arch = "Darwin" - -common = '' - -cpppath = [ - f"#/", - '#msgq/', - '/usr/lib/include', - sysconfig.get_paths()['include'], -] - -AddOption('--minimal', - action='store_false', - dest='extras', - default=True, - help='the minimum build. no tests, tools, etc.') - -AddOption('--asan', - action='store_true', - help='turn on ASAN') - -AddOption('--ubsan', - action='store_true', - help='turn on UBSan') - -ccflags = [] -ldflags = [] -if GetOption('ubsan'): - flags = [ - "-fsanitize=undefined", - "-fno-sanitize-recover=undefined", - ] - ccflags += flags - ldflags += flags -elif GetOption('asan'): - ccflags += ["-fsanitize=address", "-fno-omit-frame-pointer"] - ldflags += ["-fsanitize=address"] - -env = Environment( - ENV=os.environ, - CCFLAGS=[ - "-g", - "-fPIC", - "-O2", - "-Wunused", - "-Werror", - "-Wshadow" if arch == "Darwin" else "-Wshadow=local", - "-Wno-vla-cxx-extension", - "-Wno-unknown-warning-option", - ] + ccflags, - LDFLAGS=ldflags, - LINKFLAGS=ldflags, - - CFLAGS="-std=gnu11", - CXXFLAGS="-std=c++1z", - CPPPATH=cpppath, - CYTHONCFILESUFFIX=".cpp", - tools=["default", "cython"] -) - -Export('env', 'arch', 'common') - -envCython = env.Clone(LIBS=[]) -envCython["CPPPATH"] += [np.get_include()] -envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"] -envCython["CCFLAGS"].remove('-Werror') -if arch == "Darwin": - envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] -else: - envCython["LINKFLAGS"] = ["-pthread", "-shared"] - -Export('envCython') - - -SConscript(['SConscript']) diff --git a/build.py b/build.py new file mode 100644 index 000000000..c58a939f8 --- /dev/null +++ b/build.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Build system for msgq — replaces scons with direct clang++ subprocess calls.""" +import platform +import subprocess +import sysconfig +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).parent.resolve() +MSGQ_DIR = ROOT / "msgq" +VIPC_DIR = MSGQ_DIR / "visionipc" + +arch = subprocess.check_output(["uname", "-m"], encoding="utf8").strip() +IS_DARWIN = platform.system() == "Darwin" +if IS_DARWIN: + arch = "Darwin" + +CXX = "clang++" +AR = "ar" + +CXXFLAGS = [ + "-std=c++1z", "-g", "-fPIC", "-O2", + "-Wunused", "-Werror", + "-Wshadow" if IS_DARWIN else "-Wshadow=local", + "-Wno-vla-cxx-extension", + "-Wno-unknown-warning-option", + "-MMD", +] + +CYTHON_CXXFLAGS = [ + "-std=c++1z", "-g", "-fPIC", "-O2", + "-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", + "-Wno-deprecated-declarations", + "-Wno-unknown-warning-option", +] + +INCLUDE = [ + f"-I{ROOT}", + f"-I{MSGQ_DIR}", + "-I/usr/lib/include", + f"-I{sysconfig.get_paths()['include']}", +] + +CYTHON_INCLUDE = INCLUDE + [f"-I{np.get_include()}"] + +if IS_DARWIN: + CYTHON_LDFLAGS = ["-bundle", "-undefined", "dynamic_lookup"] +else: + CYTHON_LDFLAGS = ["-pthread", "-shared"] + + +def _parse_depfile(depfile: Path) -> list[str]: + """Parse a .d dependency file and return list of dependency paths.""" + if not depfile.exists(): + return [] + text = depfile.read_text() + # Format: "target: dep1 dep2 \\\n dep3 dep4" + text = text.replace("\\\n", " ") + # Strip "target:" prefix + _, _, deps = text.partition(":") + return deps.split() + + +def _needs_rebuild(output: Path, sources: list[Path], depfile: Path | None = None) -> bool: + """Check if output needs rebuilding based on source and dependency mtimes.""" + if not output.exists(): + return True + out_mtime = output.stat().st_mtime + for src in sources: + if src.exists() and src.stat().st_mtime > out_mtime: + return True + if depfile is not None: + for dep in _parse_depfile(depfile): + p = Path(dep) + if p.exists() and p.stat().st_mtime > out_mtime: + return True + return False + + +def _compile(src: Path, obj: Path, extra_flags: list[str] | None = None) -> bool: + """Compile a .cc to .o. Returns True if compilation ran.""" + depfile = obj.with_suffix(".d") + if not _needs_rebuild(obj, [src], depfile): + return False + cmd = [CXX] + CXXFLAGS + INCLUDE + (extra_flags or []) + ["-c", str(src), "-o", str(obj)] + subprocess.check_call(cmd, cwd=ROOT) + return True + + +def _compile_parallel(sources: list[Path], objects: list[Path]) -> bool: + """Compile multiple .cc files in parallel. Returns True if any compiled.""" + jobs = [(src, obj) for src, obj in zip(sources, objects) + if _needs_rebuild(obj, [src], obj.with_suffix(".d"))] + if not jobs: + return False + procs = [] + for src, obj in jobs: + cmd = [CXX] + CXXFLAGS + INCLUDE + ["-c", str(src), "-o", str(obj)] + procs.append((src, subprocess.Popen(cmd, cwd=ROOT))) + for src, proc in procs: + if proc.wait() != 0: + raise subprocess.CalledProcessError(proc.returncode, f"compile {src}") + return True + + +def _archive(lib: Path, objects: list[Path]) -> bool: + """Create a static library from objects. Returns True if archive ran.""" + if not _needs_rebuild(lib, objects): + return False + subprocess.check_call([AR, "rcs", str(lib)] + [str(o) for o in objects], cwd=ROOT) + return True + + +def _cythonize(pyx: Path, cpp: Path) -> bool: + """Cythonize a .pyx to .cpp. Returns True if cythonize ran.""" + # Check pyx and all .pxd files in the same directory + pxd_files = list(pyx.parent.glob("*.pxd")) + if not _needs_rebuild(cpp, [pyx] + pxd_files): + return False + subprocess.check_call(["cythonize", str(pyx)], cwd=ROOT) + return True + + +def _compile_cython_so(cpp: Path, so: Path, libs: list[Path]) -> bool: + """Compile a Cython .cpp into a .so. Returns True if compilation ran.""" + if not _needs_rebuild(so, [cpp] + libs): + return False + obj = cpp.with_suffix(".o") + subprocess.check_call( + [CXX] + CYTHON_CXXFLAGS + CYTHON_INCLUDE + ["-c", str(cpp), "-o", str(obj)], + cwd=ROOT, + ) + subprocess.check_call( + [CXX] + CYTHON_LDFLAGS + [str(obj)] + [str(l) for l in libs] + ["-o", str(so)], + cwd=ROOT, + ) + return True + + +# --- msgq --- + +MSGQ_SOURCES = [MSGQ_DIR / f for f in ["ipc.cc", "event.cc", "impl_msgq.cc", "impl_fake.cc", "msgq.cc"]] +MSGQ_OBJECTS = [s.with_suffix(".o") for s in MSGQ_SOURCES] +LIBMSGQ = ROOT / "libmsgq.a" + + +def build_msgq(): + """Build libmsgq.a and msgq/ipc_pyx.so.""" + any_compiled = _compile_parallel(MSGQ_SOURCES, MSGQ_OBJECTS) + + if any_compiled or not LIBMSGQ.exists(): + _archive(LIBMSGQ, MSGQ_OBJECTS) + + pyx = MSGQ_DIR / "ipc_pyx.pyx" + cpp = MSGQ_DIR / "ipc_pyx.cpp" + so = MSGQ_DIR / "ipc_pyx.so" + _cythonize(pyx, cpp) + _compile_cython_so(cpp, so, [LIBMSGQ]) + + +# --- visionipc --- + +VIPC_FILES = ["visionipc.cc", "visionipc_server.cc", "visionipc_client.cc"] +if arch == "larch64": + VIPC_FILES.append("visionbuf_ion.cc") +else: + VIPC_FILES.append("visionbuf.cc") + +VIPC_SOURCES = [VIPC_DIR / f for f in VIPC_FILES] +VIPC_OBJECTS = [s.with_suffix(".o") for s in VIPC_SOURCES] +LIBVIPC = ROOT / "libvisionipc.a" + + +def build_visionipc(): + """Build libvisionipc.a and msgq/visionipc/visionipc_pyx.so.""" + build_msgq() + + any_compiled = _compile_parallel(VIPC_SOURCES, VIPC_OBJECTS) + + if any_compiled or not LIBVIPC.exists(): + _archive(LIBVIPC, VIPC_OBJECTS) + + pyx = VIPC_DIR / "visionipc_pyx.pyx" + cpp = VIPC_DIR / "visionipc_pyx.cpp" + so = VIPC_DIR / "visionipc_pyx.so" + _cythonize(pyx, cpp) + _compile_cython_so(cpp, so, [LIBVIPC, LIBMSGQ]) + + +# --- test runners --- + +def build_test_runners(): + """Build C++ test executables.""" + build_visionipc() + + # msgq test runner + test_srcs = [MSGQ_DIR / "test_runner.cc", MSGQ_DIR / "msgq_tests.cc"] + test_objs = [s.with_suffix(".o") for s in test_srcs] + test_bin = MSGQ_DIR / "test_runner" + _compile_parallel(test_srcs, test_objs) + if _needs_rebuild(test_bin, test_objs + [LIBMSGQ]): + subprocess.check_call( + [CXX] + [str(o) for o in test_objs] + [str(LIBMSGQ), "-o", str(test_bin)], + cwd=ROOT, + ) + + # visionipc test runner + vipc_test_srcs = [VIPC_DIR / "test_runner.cc", VIPC_DIR / "visionipc_tests.cc"] + vipc_test_objs = [s.with_suffix(".o") for s in vipc_test_srcs] + vipc_test_bin = VIPC_DIR / "test_runner" + _compile_parallel(vipc_test_srcs, vipc_test_objs) + if _needs_rebuild(vipc_test_bin, vipc_test_objs + [LIBVIPC, LIBMSGQ]): + subprocess.check_call( + [CXX] + [str(o) for o in vipc_test_objs] + [str(LIBVIPC), str(LIBMSGQ), "-lpthread", "-o", str(vipc_test_bin)], + cwd=ROOT, + ) + + +def build(): + """Build everything.""" + build_visionipc() + + +if __name__ == "__main__": + build() + build_test_runners() diff --git a/msgq/.gitignore b/msgq/.gitignore index 6bd751773..a49eb491e 100644 --- a/msgq/.gitignore +++ b/msgq/.gitignore @@ -1 +1,2 @@ ipc_pyx.cpp +visionipc/visionipc_pyx.cpp diff --git a/msgq/__init__.py b/msgq/__init__.py index 574e100a8..8ddeb166d 100644 --- a/msgq/__init__.py +++ b/msgq/__init__.py @@ -1,4 +1,6 @@ -# must be built with scons +from build import build_msgq +build_msgq() + from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event from msgq.ipc_pyx import MultiplePublishersError, IpcError diff --git a/msgq/visionipc/__init__.py b/msgq/visionipc/__init__.py index 57011537e..d7e693cfa 100644 --- a/msgq/visionipc/__init__.py +++ b/msgq/visionipc/__init__.py @@ -1,3 +1,6 @@ +from build import build_visionipc +build_visionipc() + from msgq.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, VisionStreamType, get_endpoint_name assert VisionBuf assert VisionIpcClient diff --git a/pyproject.toml b/pyproject.toml index efbb89ad7..eff3f54e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers = [ dependencies = [ "setuptools", # for distutils "Cython", - "scons", + "ruff", "parameterized", "coverage", @@ -41,9 +41,6 @@ target-version="py311" "pytest.main".msg = "pytest.main requires special handling that is easy to mess up!" "unittest".msg = "Use pytest" -[tool.ty.src] -exclude = ["site_scons/"] - [tool.ty.rules] # Cython modules are compiled at build time, not available for static analysis unresolved-import = "ignore" diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py deleted file mode 100644 index c29147553..000000000 --- a/site_scons/site_tools/cython.py +++ /dev/null @@ -1,72 +0,0 @@ -import re -import SCons -from SCons.Action import Action -from SCons.Scanner import Scanner - -pyx_from_import_re = re.compile(r'^from\s+(\S+)\s+cimport', re.M) -pyx_import_re = re.compile(r'^cimport\s+(\S+)', re.M) -cdef_import_re = re.compile(r'^cdef extern from\s+.(\S+).:', re.M) - - -def pyx_scan(node, env, path, arg=None): - contents = node.get_text_contents() - - # from cimport ... - matches = pyx_from_import_re.findall(contents) - # cimport - matches += pyx_import_re.findall(contents) - - # Modules can be either .pxd or .pyx files - files = [m.replace('.', '/') + '.pxd' for m in matches] - files += [m.replace('.', '/') + '.pyx' for m in matches] - - # cdef extern from - files += cdef_import_re.findall(contents) - - # Handle relative imports - cur_dir = str(node.get_dir()) - files = [cur_dir + f if f.startswith('/') else f for f in files] - - # Filter out non-existing files (probably system imports) - files = [f for f in files if env.File(f).exists()] - return env.File(files) - - -pyxscanner = Scanner(function=pyx_scan, skeys=['.pyx', '.pxd'], recursive=True) -cythonAction = Action("$CYTHONCOM") - - -def create_builder(env): - try: - cython = env['BUILDERS']['Cython'] - except KeyError: - cython = SCons.Builder.Builder( - action=cythonAction, - emitter={}, - suffix=cython_suffix_emitter, - single_source=1 - ) - env.Append(SCANNERS=pyxscanner) - env['BUILDERS']['Cython'] = cython - return cython - -def cython_suffix_emitter(env, source): - return "$CYTHONCFILESUFFIX" - -def generate(env): - env["CYTHON"] = "cythonize" - env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" - env["CYTHONCFILESUFFIX"] = ".cpp" - - c_file, _ = SCons.Tool.createCFileBuilders(env) - - c_file.suffix['.pyx'] = cython_suffix_emitter - c_file.add_action('.pyx', cythonAction) - - c_file.suffix['.py'] = cython_suffix_emitter - c_file.add_action('.py', cythonAction) - - create_builder(env) - -def exists(env): - return True diff --git a/test.sh b/test.sh index d91afb035..6bd67a99a 100755 --- a/test.sh +++ b/test.sh @@ -7,7 +7,7 @@ cd $DIR source ./setup.sh # *** build *** -scons -j8 +python build.py # *** lint + test *** lefthook run test From e12228191c19deb9db90609a96230dea3054fb5c Mon Sep 17 00:00:00 2001 From: Matt Purnell Date: Thu, 12 Mar 2026 23:52:36 -0500 Subject: [PATCH 2/2] build.py: support EXTRA_CXXFLAGS env var Allows callers (e.g. openpilot's scons) to pass additional compiler flags like -DSWAGLOG for swaglog integration. Co-Authored-By: Claude Opus 4.6 --- build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index c58a939f8..7c2bec3f7 100644 --- a/build.py +++ b/build.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Build system for msgq — replaces scons with direct clang++ subprocess calls.""" +import os import platform import subprocess import sysconfig @@ -19,6 +20,8 @@ CXX = "clang++" AR = "ar" +EXTRA_CXXFLAGS = os.environ.get("EXTRA_CXXFLAGS", "").split() + CXXFLAGS = [ "-std=c++1z", "-g", "-fPIC", "-O2", "-Wunused", "-Werror", @@ -26,7 +29,7 @@ "-Wno-vla-cxx-extension", "-Wno-unknown-warning-option", "-MMD", -] +] + EXTRA_CXXFLAGS CYTHON_CXXFLAGS = [ "-std=c++1z", "-g", "-fPIC", "-O2",