From 813c9e7a65ddf021132c577608abe9195c6e5217 Mon Sep 17 00:00:00 2001 From: Nicholas Chaimov Date: Wed, 7 Oct 2026 13:00:31 -0700 Subject: [PATCH 01/18] Fix signal registration before crash handler init --- src/client/crash_handler/crash_handler.c | 1 + src/client/crash_handler/crash_sigchain.c | 10 ++++++++++ src/client/crash_handler/crash_sigchain.h | 1 + testsuite/run_crash_tests_template.sh | 9 ++++++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/client/crash_handler/crash_handler.c b/src/client/crash_handler/crash_handler.c index ba521d9c..76fcd924 100644 --- a/src/client/crash_handler/crash_handler.c +++ b/src/client/crash_handler/crash_handler.c @@ -434,6 +434,7 @@ int crash_handler_install(int global_rank, int ldcsid_in) sigaddset(&sa.sa_mask, SIGABRT); const int sigs[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT }; + crash_sigchain_activate(); for (size_t i = 0; i < sizeof(sigs) / sizeof(sigs[0]); i++) { /* If a signal handler was already installed by the time Spindle registers its signal handler, we save it to use in diff --git a/src/client/crash_handler/crash_sigchain.c b/src/client/crash_handler/crash_sigchain.c index 7e4351e5..c9de0f33 100644 --- a/src/client/crash_handler/crash_sigchain.c +++ b/src/client/crash_handler/crash_sigchain.c @@ -72,8 +72,18 @@ struct app_disposition { static const int OWNED_SIGS[] = { SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT }; +/* Set once the crash handler has installed its handlers. */ +static int sigchain_active = 0; + +void crash_sigchain_activate(void) +{ + sigchain_active = 1; +} + int crash_sigchain_is_owned(int sig) { + if (!sigchain_active) + return 0; for (size_t i = 0; i < sizeof(OWNED_SIGS) / sizeof(OWNED_SIGS[0]); i++) { if (OWNED_SIGS[i] == sig) return 1; diff --git a/src/client/crash_handler/crash_sigchain.h b/src/client/crash_handler/crash_sigchain.h index d6d2fa39..273dcfaf 100644 --- a/src/client/crash_handler/crash_sigchain.h +++ b/src/client/crash_handler/crash_sigchain.h @@ -26,6 +26,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #define CRASH_CHAIN_HANDLED 1 /* application handler ran and returned */ #define CRASH_CHAIN_IGNORED 2 /* application disposition is SIG_IGN */ +void crash_sigchain_activate(void); int crash_sigchain_is_owned(int sig); void crash_sigchain_register_existing_handler(int sig, const struct sigaction *handler_old); int crash_sigchain_chain_to_app(int sig, siginfo_t *info, void *ucontext); diff --git a/testsuite/run_crash_tests_template.sh b/testsuite/run_crash_tests_template.sh index 1e8e2bf9..03b04379 100755 --- a/testsuite/run_crash_tests_template.sh +++ b/testsuite/run_crash_tests_template.sh @@ -35,6 +35,7 @@ die() { echo "FAIL: $*" >&2; exit 1; } # flags (comma-separated): "multi-rank" skips the mode on a single rank; # "clean" expects the test to NOT crash; "altstack" runs the mode with # --crash-altstack; "nofollowfork" runs it with --follow-fork=false; +# "nocrashhandler" runs it without the crash handler; # "forkchild" means rows may come from fork or exec children of a rank. # top_frame_regex: regex that should match the top frame in produced coredumps. # Note that all threads will be checked, so in multithreaded examples the regex should @@ -70,6 +71,7 @@ CRASH_TESTS=( 'safepoint-fix-write ; 0 ; 0 ; clean ; -' 'safepoint-fix-write-altstack ; 0 ; 0 ; clean,altstack ; - ; ; ; safepoint-fix-write' 'safepoint-longjmp ; 0 ; 0 ; clean ; -' + 'safepoint-nocrashhandler ; 0 ; 0 ; clean,nocrashhandler ; - ; ; ; safepoint' 'safepoint-span-read ; 0 ; 0 ; clean ; -' 'safepoint-span-write ; 0 ; 0 ; clean ; -' 'safepoint-span-bad-read ; 1 ; N ; ; do_safepoint_span_bad_read' @@ -266,6 +268,11 @@ launch() { cmdline_opts="$cmdline_opts --follow-fork=false" flux_opts+=(-o spindle.follow-fork=no) fi + if has_flag "$mode" nocrashhandler; then + cmdline_opts="" + flux_opts=(-o spindle) + export SPINDLE_NO_CRASH_DEDUP=1 + fi ulimit -c unlimited # Make libcrashfuncs.so visible to dlopen() from the mode's scratch dir. export LD_LIBRARY_PATH="$TESTDIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" @@ -290,7 +297,7 @@ launch() { ;; slurm-plugin) salloc -N"$NODES" -n"$NODES" \ - srun --spindle="$cmdline_opts" \ + srun "--spindle${cmdline_opts:+=$cmdline_opts}" \ "$binary" --crash-mode "$crash_mode" ;; esac From b9e9aaf3db9ba0c1c513d275b4a3f118966875ff Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Wed, 21 Oct 2026 16:44:16 -0700 Subject: [PATCH 02/18] config_helper: Initial commit. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ● Add spindle-config-helper utility (milestone 1) Introduces a new tool to help users generate and validate Spindle configuration files. This milestone implements the foundational parse-and-output functionality. Features: - Parses configuration from command line, environment variables, and config files using existing config_mgr.cc infrastructure - Outputs merged configuration in spindle.conf format - Supports all 62 Spindle configuration options - Command-line help: -h/--help, -V/--version, --usage - Application argument is optional (auto-supplies dummy if needed) - Man page: spindle-config-helper(1) Architecture: - New binary: src/config_helper/spindle-config-helper - Private static library: src/fe/startup/libconfigparse.a Contains config_mgr.cc, config_parser.cc, parseargs.cc, and supporting utilities. Library is not installed (noinst_LIBRARIES) to avoid distribution complexity in sidecar scenarios. - Integrated into autotools build system Future milestones will add environment probing (detecting local mount points, resource managers, security subsystems) and configuration validation. Co-Authored-By: Claude --- Makefile.am | 2 +- Makefile.in | 8 +- configure | 3 +- configure.ac | 2 +- src/config_helper/Makefile.am | 19 + src/config_helper/Makefile.in | 796 ++++++++++++++++++ src/config_helper/man/spindle-config-helper.1 | 116 +++ src/config_helper/spindle-config-helper.cc | 169 ++++ src/fe/startup/Makefile.am | 8 + src/fe/startup/Makefile.in | 189 ++++- 10 files changed, 1286 insertions(+), 26 deletions(-) create mode 100644 src/config_helper/Makefile.am create mode 100644 src/config_helper/Makefile.in create mode 100644 src/config_helper/man/spindle-config-helper.1 create mode 100644 src/config_helper/spindle-config-helper.cc diff --git a/Makefile.am b/Makefile.am index fb8e51d0..dc5d1ffe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,4 @@ -SUBDIRS = src/client src/server src/fe doc +SUBDIRS = src/client src/server src/fe src/config_helper doc if BLD_SLURMPLUGIN SUBDIRS += src/slurm_plugin endif diff --git a/Makefile.in b/Makefile.in index 9e4b21b3..c4a8089c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -166,8 +166,8 @@ am__define_uniq_tagged_files = \ ETAGS = etags CTAGS = ctags CSCOPE = cscope -DIST_SUBDIRS = src/client src/server src/fe doc src/slurm_plugin \ - src/flux testsuite +DIST_SUBDIRS = src/client src/server src/fe src/config_helper doc \ + src/slurm_plugin src/flux testsuite am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in \ $(top_srcdir)/scripts/ar-lib $(top_srcdir)/scripts/compile \ $(top_srcdir)/scripts/config.guess \ @@ -380,8 +380,8 @@ target_vendor = @target_vendor@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ -SUBDIRS = src/client src/server src/fe doc $(am__append_1) \ - $(am__append_2) $(am__append_3) +SUBDIRS = src/client src/server src/fe src/config_helper doc \ + $(am__append_1) $(am__append_2) $(am__append_3) all: config.h $(MAKE) $(AM_MAKEFLAGS) all-recursive diff --git a/configure b/configure index 0b4da648..43bf8dd5 100755 --- a/configure +++ b/configure @@ -5775,7 +5775,7 @@ $as_echo "$ac_cv_path_SED" >&6; } ac_config_headers="$ac_config_headers config.h" -ac_config_files="$ac_config_files src/flux/Makefile src/slurm_plugin/Makefile doc/Makefile testsuite/Makefile Makefile" +ac_config_files="$ac_config_files src/flux/Makefile src/slurm_plugin/Makefile src/config_helper/Makefile doc/Makefile testsuite/Makefile Makefile" { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ln -s works" >&5 $as_echo_n "checking whether ln -s works... " >&6; } @@ -20131,6 +20131,7 @@ do "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; "src/flux/Makefile") CONFIG_FILES="$CONFIG_FILES src/flux/Makefile" ;; "src/slurm_plugin/Makefile") CONFIG_FILES="$CONFIG_FILES src/slurm_plugin/Makefile" ;; + "src/config_helper/Makefile") CONFIG_FILES="$CONFIG_FILES src/config_helper/Makefile" ;; "doc/Makefile") CONFIG_FILES="$CONFIG_FILES doc/Makefile" ;; "testsuite/Makefile") CONFIG_FILES="$CONFIG_FILES testsuite/Makefile" ;; "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; diff --git a/configure.ac b/configure.ac index b2c519d4..e778fc91 100644 --- a/configure.ac +++ b/configure.ac @@ -9,7 +9,7 @@ AC_PROG_CC AC_PROG_CXX AC_PROG_SED AC_CONFIG_HEADERS([config.h]) -AC_CONFIG_FILES([src/flux/Makefile src/slurm_plugin/Makefile doc/Makefile testsuite/Makefile Makefile]) +AC_CONFIG_FILES([src/flux/Makefile src/slurm_plugin/Makefile src/config_helper/Makefile doc/Makefile testsuite/Makefile Makefile]) AC_PROG_LN_S LT_INIT diff --git a/src/config_helper/Makefile.am b/src/config_helper/Makefile.am new file mode 100644 index 00000000..45ba4128 --- /dev/null +++ b/src/config_helper/Makefile.am @@ -0,0 +1,19 @@ +bin_PROGRAMS = spindle-config-helper + +spindle_config_helper_SOURCES = spindle-config-helper.cc + +spindle_config_helper_CPPFLAGS = -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/fe/startup \ + -I$(top_srcdir)/src/fe/logging \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/server/cache \ + -I$(top_srcdir)/src/server/comlib \ + -I$(top_srcdir)/src/cobo \ + -I$(top_srcdir)/src/fe/comlib \ + -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" + +spindle_config_helper_LDADD = $(top_builddir)/src/fe/startup/libconfigparse.a \ + $(top_builddir)/src/fe/logging/libspindleflogc.la \ + -lpthread + +man_MANS = man/spindle-config-helper.1 diff --git a/src/config_helper/Makefile.in b/src/config_helper/Makefile.in new file mode 100644 index 00000000..08d47f8c --- /dev/null +++ b/src/config_helper/Makefile.in @@ -0,0 +1,796 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +bin_PROGRAMS = spindle-config-helper$(EXEEXT) +subdir = src/config_helper +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/pkg.m4 $(top_srcdir)/configure.common.ac \ + $(top_srcdir)/m4/ax_tls.m4 $(top_srcdir)/m4/lmon.m4 \ + $(top_srcdir)/m4/lx_find_mpi.m4 $(top_srcdir)/m4/runconf.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)" +PROGRAMS = $(bin_PROGRAMS) +am_spindle_config_helper_OBJECTS = \ + spindle_config_helper-spindle-config-helper.$(OBJEXT) +spindle_config_helper_OBJECTS = $(am_spindle_config_helper_OBJECTS) +spindle_config_helper_DEPENDENCIES = \ + $(top_builddir)/src/fe/startup/libconfigparse.a \ + $(top_builddir)/src/fe/logging/libspindleflogc.la +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/scripts/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = \ + ./$(DEPDIR)/spindle_config_helper-spindle-config-helper.Po +am__mv = mv -f +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CXXFLAGS) $(CXXFLAGS) +AM_V_CXX = $(am__v_CXX_@AM_V@) +am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) +am__v_CXX_0 = @echo " CXX " $@; +am__v_CXX_1 = +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) +am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) +am__v_CXXLD_0 = @echo " CXXLD " $@; +am__v_CXXLD_1 = +SOURCES = $(spindle_config_helper_SOURCES) +DIST_SOURCES = $(spindle_config_helper_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +man1dir = $(mandir)/man1 +NROFF = nroff +MANS = $(man_MANS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/scripts/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AR_FLAGS = @AR_FLAGS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BE_CC = @BE_CC@ +BE_CFLAGS = @BE_CFLAGS@ +BE_CPP = @BE_CPP@ +BE_CPPFLAGS = @BE_CPPFLAGS@ +BE_CXX = @BE_CXX@ +BE_CXXCPP = @BE_CXXCPP@ +BE_CXXFLAGS = @BE_CXXFLAGS@ +BE_LDFLAGS = @BE_LDFLAGS@ +BE_LIBS = @BE_LIBS@ +BE_host = @BE_host@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLUX_CORE_CFLAGS = @FLUX_CORE_CFLAGS@ +FLUX_CORE_LIBS = @FLUX_CORE_LIBS@ +FLUX_HOSTLIST_CFLAGS = @FLUX_HOSTLIST_CFLAGS@ +FLUX_HOSTLIST_LIBS = @FLUX_HOSTLIST_LIBS@ +GREP = @GREP@ +HAVE_LMON = @HAVE_LMON@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LAUNCHMON_INC = @LAUNCHMON_INC@ +LAUNCHMON_LIB = @LAUNCHMON_LIB@ +LAUNCHMON_RMCOMM = @LAUNCHMON_RMCOMM@ +LAUNCHMON_STATIC_LIBS = @LAUNCHMON_STATIC_LIBS@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MPICC = @MPICC@ +MPICXX = @MPICXX@ +MPI_CFLAGS = @MPI_CFLAGS@ +MPI_CLDFLAGS = @MPI_CLDFLAGS@ +MPI_CXXFLAGS = @MPI_CXXFLAGS@ +MPI_CXXLDFLAGS = @MPI_CXXLDFLAGS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PKGSYSCONF_DIR = @PKGSYSCONF_DIR@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +RANLIB = @RANLIB@ +SCONTROL_ABSPATH = @SCONTROL_ABSPATH@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SINFO_ABSPATH = @SINFO_ABSPATH@ +SRUN_PATH = @SRUN_PATH@ +STATICFLAG = @STATICFLAG@ +STRIP = @STRIP@ +TESTRM = @TESTRM@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +fluxplugin_ldflags = @fluxplugin_ldflags@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +shell_plugindir = @shell_plugindir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +spindle_config_helper_SOURCES = spindle-config-helper.cc +spindle_config_helper_CPPFLAGS = -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/src/fe/startup \ + -I$(top_srcdir)/src/fe/logging \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/server/cache \ + -I$(top_srcdir)/src/server/comlib \ + -I$(top_srcdir)/src/cobo \ + -I$(top_srcdir)/src/fe/comlib \ + -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" + +spindle_config_helper_LDADD = $(top_builddir)/src/fe/startup/libconfigparse.a \ + $(top_builddir)/src/fe/logging/libspindleflogc.la \ + -lpthread + +man_MANS = man/spindle-config-helper.1 +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/config_helper/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/config_helper/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \ + fi; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p \ + || test -f $$p1 \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' \ + `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +spindle-config-helper$(EXEEXT): $(spindle_config_helper_OBJECTS) $(spindle_config_helper_DEPENDENCIES) $(EXTRA_spindle_config_helper_DEPENDENCIES) + @rm -f spindle-config-helper$(EXEEXT) + $(AM_V_CXXLD)$(CXXLINK) $(spindle_config_helper_OBJECTS) $(spindle_config_helper_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_config_helper-spindle-config-helper.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.cc.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ +@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< + +spindle_config_helper-spindle-config-helper.o: spindle-config-helper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_config_helper-spindle-config-helper.o -MD -MP -MF $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo -c -o spindle_config_helper-spindle-config-helper.o `test -f 'spindle-config-helper.cc' || echo '$(srcdir)/'`spindle-config-helper.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo $(DEPDIR)/spindle_config_helper-spindle-config-helper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='spindle-config-helper.cc' object='spindle_config_helper-spindle-config-helper.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_config_helper-spindle-config-helper.o `test -f 'spindle-config-helper.cc' || echo '$(srcdir)/'`spindle-config-helper.cc + +spindle_config_helper-spindle-config-helper.obj: spindle-config-helper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_config_helper-spindle-config-helper.obj -MD -MP -MF $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo -c -o spindle_config_helper-spindle-config-helper.obj `if test -f 'spindle-config-helper.cc'; then $(CYGPATH_W) 'spindle-config-helper.cc'; else $(CYGPATH_W) '$(srcdir)/spindle-config-helper.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo $(DEPDIR)/spindle_config_helper-spindle-config-helper.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='spindle-config-helper.cc' object='spindle_config_helper-spindle-config-helper.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_config_helper-spindle-config-helper.obj `if test -f 'spindle-config-helper.cc'; then $(CYGPATH_W) 'spindle-config-helper.cc'; else $(CYGPATH_W) '$(srcdir)/spindle-config-helper.cc'; fi` + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-man1: $(man_MANS) + @$(NORMAL_INSTALL) + @list1=''; \ + list2='$(man_MANS)'; \ + test -n "$(man1dir)" \ + && test -n "`echo $$list1$$list2`" \ + || exit 0; \ + echo " $(MKDIR_P) '$(DESTDIR)$(man1dir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(man1dir)" || exit 1; \ + { for i in $$list1; do echo "$$i"; done; \ + if test -n "$$list2"; then \ + for i in $$list2; do echo "$$i"; done \ + | sed -n '/\.1[a-z]*$$/p'; \ + fi; \ + } | while read p; do \ + if test -f $$p; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; echo "$$p"; \ + done | \ + sed -e 'n;s,.*/,,;p;h;s,.*\.,,;s,^[^1][0-9a-z]*$$,1,;x' \ + -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,' | \ + sed 'N;N;s,\n, ,g' | { \ + list=; while read file base inst; do \ + if test "$$base" = "$$inst"; then list="$$list $$file"; else \ + echo " $(INSTALL_DATA) '$$file' '$(DESTDIR)$(man1dir)/$$inst'"; \ + $(INSTALL_DATA) "$$file" "$(DESTDIR)$(man1dir)/$$inst" || exit $$?; \ + fi; \ + done; \ + for i in $$list; do echo "$$i"; done | $(am__base_list) | \ + while read files; do \ + test -z "$$files" || { \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(man1dir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(man1dir)" || exit $$?; }; \ + done; } + +uninstall-man1: + @$(NORMAL_UNINSTALL) + @list=''; test -n "$(man1dir)" || exit 0; \ + files=`{ for i in $$list; do echo "$$i"; done; \ + l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \ + sed -n '/\.1[a-z]*$$/p'; \ + } | sed -e 's,.*/,,;h;s,.*\.,,;s,^[^1][0-9a-z]*$$,1,;x' \ + -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,'`; \ + dir='$(DESTDIR)$(man1dir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(PROGRAMS) $(MANS) +installdirs: + for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-binPROGRAMS clean-generic clean-libtool mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/spindle_config_helper-spindle-config-helper.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-man + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-binPROGRAMS + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: install-man1 + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/spindle_config_helper-spindle-config-helper.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS uninstall-man + +uninstall-man: uninstall-man1 + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-binPROGRAMS clean-generic clean-libtool cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-binPROGRAMS \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-man1 \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-binPROGRAMS uninstall-man uninstall-man1 + +.PRECIOUS: Makefile + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/config_helper/man/spindle-config-helper.1 b/src/config_helper/man/spindle-config-helper.1 new file mode 100644 index 00000000..f275d3e0 --- /dev/null +++ b/src/config_helper/man/spindle-config-helper.1 @@ -0,0 +1,116 @@ +.TH SPINDLE-CONFIG-HELPER 1 "October 2026" "Spindle 0.13" "User Commands" +.SH NAME +spindle-config-helper \- generate and validate Spindle configuration files +.SH SYNOPSIS +.B spindle-config-helper +[\fIOPTIONS\fR] +.SH DESCRIPTION +.B spindle-config-helper +is a utility for generating and validating Spindle configuration files. +It parses configuration from command-line options, environment variables, and +existing configuration files, then outputs the resulting configuration in +.I spindle.conf +format. +.PP +In future releases, this tool will also probe the runtime environment to +automatically detect optimal configuration settings, including: +.IP \(bu 2 +Local mount points suitable for cache and communication paths +.IP \(bu 2 +Available resource managers (Slurm, Flux, LSF) +.IP \(bu 2 +Security subsystems (Munge availability) +.IP \(bu 2 +MPI environment characteristics +.PP +The tool is designed to be run on compute nodes in the same environment where +Spindle will execute, as login/front-end nodes may have different characteristics +than compute nodes. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Display usage information and exit. +.TP +.BR \-V ", " \-\-version +Display version information and exit. +.TP +.BR \-\-usage +Display a brief usage summary with all available options and exit. +.PP +All Spindle configuration options are accepted. Use +.B --usage +to see a complete list of available options, or see the Spindle documentation +for detailed descriptions of each option. +.SH EXAMPLES +.TP +Parse command-line options and output configuration: +.nf +.RS +spindle-config-helper --commpaths=/dev/shm --cachepaths=/tmp/cache +.RE +.fi +.TP +Parse an existing configuration file: +.nf +.RS +spindle-config-helper --config=/etc/spindle.conf +.RE +.fi +.TP +Combine multiple configuration sources: +.nf +.RS +export SPINDLE_SECURITY=none +spindle-config-helper --config=/etc/spindle.conf --launcher=slurm +.RE +.fi +.PP +The tool merges configuration from multiple sources in the following order +(later sources override earlier ones): +.IP 1. 3 +Built-in defaults +.IP 2. 3 +System configuration file +.IP 3. 3 +User configuration file +.IP 4. 3 +Environment variables +.IP 5. 3 +Command-line options +.SH FUTURE FUNCTIONALITY +Future versions will support: +.TP +.B Environment Probing +Automatically detect and recommend optimal settings for cache paths, communication +paths, and other configuration based on the compute node environment. +.TP +.B Configuration Validation +Verify that specified paths exist, are writable, and are on appropriate filesystem +types (e.g., local vs. network filesystems). +.TP +.B Performance Tuning +Analyze the environment and suggest performance-oriented configuration settings. +.TP +.B MPI Integration +Run as an MPI application to gather environment information across all compute nodes +in an allocation. +.SH FILES +.TP +.I /etc/spindle.conf +System-wide Spindle configuration file. +.TP +.I ~/.spindle/spindle.conf +Per-user Spindle configuration file. +.SH EXIT STATUS +.TP +.B 0 +Success. Valid configuration was parsed and output. +.TP +.B 1 +Error. Invalid configuration or parse error occurred. +.SH SEE ALSO +.BR spindle (1) +.SH AUTHOR +The Spindle development team at Lawrence Livermore National Laboratory. +.SH BUGS +Report bugs to the Spindle issue tracker at https://github.com/hpc/Spindle/issues diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc new file mode 100644 index 00000000..c1237bca --- /dev/null +++ b/src/config_helper/spindle-config-helper.cc @@ -0,0 +1,169 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU Lesser General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "config.h" +#include "config_mgr.h" +#include +#include +#include +#include + +static void print_version() +{ + std::cout << "spindle-config-helper version " << PACKAGE_VERSION << std::endl; +} + +static void print_usage(const char *progname) +{ + std::cout << "Usage: " << progname << " [OPTIONS]\n\n" + << "Parse Spindle configuration from command line, environment variables, and\n" + << "configuration files, then output the resulting configuration in spindle.conf format.\n\n" + << "Options:\n" + << " -h, --help Display this help message and exit\n" + << " -V, --version Display version information and exit\n" + << " --usage Display brief usage summary and exit\n\n" + << "All Spindle configuration options are supported. See spindle-config-helper(1)\n" + << "for details, or use --usage for a complete list of options.\n\n" + << "Examples:\n" + << " " << progname << " --commpaths=/dev/shm --cachepaths=/tmp\n" + << " " << progname << " --config=/etc/spindle.conf\n" + << " " << progname << " --launcher=slurm --security=none\n" + << std::endl; +} + +static void format_config_value(const ConfigMap &confmap, SpindleConfigID id, + const SpindleOption &opt) +{ + if (!confmap.isSet(id)) { + return; + } + + std::string value; + bool has_value = false; + + switch (opt.conftype) { + case cvBool: { + auto result = confmap.getValueBool(id); + if (result.first) { + value = result.second ? "yes" : "no"; + has_value = true; + } + break; + } + case cvInteger: { + auto result = confmap.getValueIntegral(id); + if (result.first) { + value = std::to_string(result.second); + has_value = true; + } + break; + } + case cvString: + case cvStringOptional: + case cvList: + case cvEnum: { + auto result = confmap.getValueString(id); + if (result.first) { + value = result.second; + has_value = true; + } + break; + } + } + + if (has_value && opt.cmdline_long) { + // Convert cmdline option name to config file format + // e.g., "comm-paths" -> "commpaths" + std::string key = opt.cmdline_long; + // Remove dashes using erase-remove idiom + key.erase(std::remove_if(key.begin(), key.end(), + [](char c) { return c == '-'; }), + key.end()); + + std::cout << key << " = " << value << std::endl; + } +} + +int main(int argc, char *argv[]) +{ + // Handle -h/--help and -V/--version before config parsing + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + print_usage(argv[0]); + return 0; + } + if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) { + print_version(); + return 0; + } + } + + // Check if application command was provided (after '--') + bool has_app = false; + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--") == 0 && i + 1 < argc) { + has_app = true; + break; + } + } + + // If no application provided, add a dummy one for the parser + char **modified_argv = argv; + int modified_argc = argc; + char *dummy_app = nullptr; + + if (!has_app) { + dummy_app = strdup("/bin/true"); + modified_argv = new char*[argc + 2]; + for (int i = 0; i < argc; i++) { + modified_argv[i] = argv[i]; + } + modified_argv[argc] = strdup("--"); + modified_argv[argc + 1] = dummy_app; + modified_argc = argc + 2; + } + + ConfigMap confmap; + std::string errmsg; + + // Reuse Spindle's existing parsing: cmdline, env vars, config files + // Pass false for inExecutable since we're not the spindle launcher + if (!gatherAllConfigInfo(modified_argc, modified_argv, false, confmap, errmsg)) { + std::cerr << "Error: " << errmsg << std::endl; + if (!has_app) { + free(dummy_app); + free(modified_argv[argc]); + delete[] modified_argv; + } + return 1; + } + + // Clean up temporary argv if we created one + if (!has_app) { + free(dummy_app); + free(modified_argv[argc]); + delete[] modified_argv; + } + + // Output configuration in spindle.conf format + const std::map &options = getOptionsByID(); + + for (const auto &pair : options) { + format_config_value(confmap, pair.first, pair.second); + } + + return 0; +} diff --git a/src/fe/startup/Makefile.am b/src/fe/startup/Makefile.am index c4daceb3..82582e89 100644 --- a/src/fe/startup/Makefile.am +++ b/src/fe/startup/Makefile.am @@ -1,6 +1,7 @@ bin_PROGRAMS = spindle lib_LTLIBRARIES = libspindlefe.la noinst_LTLIBRARIES = libspindlefe_static.la +noinst_LIBRARIES = libconfigparse.a include_HEADERS = $(top_srcdir)/../include/spindle_launch.h @@ -43,3 +44,10 @@ if LMON spindle_LDADD += $(top_builddir)/launchmon/libfelmon.la endif +# Private static library for config parsing (used by spindle-config-helper) +libconfigparse_a_SOURCES = config_mgr.cc config_parser.cc parseargs.cc \ + $(top_srcdir)/../utils/pathfn.c \ + $(top_srcdir)/../utils/keyfile.c \ + $(top_srcdir)/../utils/parseloc.c +libconfigparse_a_CPPFLAGS = $(CORE_CPPFLAGS) + diff --git a/src/fe/startup/Makefile.in b/src/fe/startup/Makefile.in index 0f678858..9ed31186 100644 --- a/src/fe/startup/Makefile.in +++ b/src/fe/startup/Makefile.in @@ -16,6 +16,7 @@ + VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ @@ -121,6 +122,7 @@ CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(libdir)" \ "$(DESTDIR)$(includedir)" PROGRAMS = $(bin_PROGRAMS) +LIBRARIES = $(noinst_LIBRARIES) am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; am__vpath_adj = case $$p in \ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ @@ -149,12 +151,26 @@ am__uninstall_files_from_dir = { \ $(am__cd) "$$dir" && rm -f $$files; }; \ } LTLIBRARIES = $(lib_LTLIBRARIES) $(noinst_LTLIBRARIES) +ARFLAGS = cru +AM_V_AR = $(am__v_AR_@AM_V@) +am__v_AR_ = $(am__v_AR_@AM_DEFAULT_V@) +am__v_AR_0 = @echo " AR " $@; +am__v_AR_1 = +libconfigparse_a_AR = $(AR) $(ARFLAGS) +libconfigparse_a_LIBADD = +am__dirstamp = $(am__leading_dot)dirstamp +am_libconfigparse_a_OBJECTS = libconfigparse_a-config_mgr.$(OBJEXT) \ + libconfigparse_a-config_parser.$(OBJEXT) \ + libconfigparse_a-parseargs.$(OBJEXT) \ + $(top_builddir)/../utils/libconfigparse_a-pathfn.$(OBJEXT) \ + $(top_builddir)/../utils/libconfigparse_a-keyfile.$(OBJEXT) \ + $(top_builddir)/../utils/libconfigparse_a-parseloc.$(OBJEXT) +libconfigparse_a_OBJECTS = $(am_libconfigparse_a_OBJECTS) am__DEPENDENCIES_1 = am__DEPENDENCIES_2 = $(top_builddir)/logging/libspindleflogc.la \ $(am__append_1) $(am__append_2) $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) libspindlefe_la_DEPENDENCIES = $(am__DEPENDENCIES_2) -am__dirstamp = $(am__leading_dot)dirstamp am__objects_1 = libspindlefe_la-spindle_fe.lo \ libspindlefe_la-parseargs.lo libspindlefe_la-config_parser.lo \ libspindlefe_la-config_mgr.lo libspindlefe_la-parse_preload.lo \ @@ -235,7 +251,10 @@ am__v_at_1 = DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/../../scripts/depcomp am__maybe_remake_depfiles = depfiles -am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo \ +am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Po \ + $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Po \ + $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Po \ + $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo \ $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-parseloc.Plo \ $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Plo \ $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-rshlaunch.Plo \ @@ -247,6 +266,9 @@ am__depfiles_remade = $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile $(top_builddir)/../utils/$(DEPDIR)/spindle-parseloc.Po \ $(top_builddir)/../utils/$(DEPDIR)/spindle-pathfn.Po \ $(top_builddir)/../utils/$(DEPDIR)/spindle-rshlaunch.Po \ + ./$(DEPDIR)/libconfigparse_a-config_mgr.Po \ + ./$(DEPDIR)/libconfigparse_a-config_parser.Po \ + ./$(DEPDIR)/libconfigparse_a-parseargs.Po \ ./$(DEPDIR)/libspindlefe_la-config_mgr.Plo \ ./$(DEPDIR)/libspindlefe_la-config_parser.Plo \ ./$(DEPDIR)/libspindlefe_la-parse_preload.Plo \ @@ -308,9 +330,9 @@ AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) am__v_CXXLD_0 = @echo " CXXLD " $@; am__v_CXXLD_1 = -SOURCES = $(libspindlefe_la_SOURCES) $(libspindlefe_static_la_SOURCES) \ - $(spindle_SOURCES) -DIST_SOURCES = $(libspindlefe_la_SOURCES) \ +SOURCES = $(libconfigparse_a_SOURCES) $(libspindlefe_la_SOURCES) \ + $(libspindlefe_static_la_SOURCES) $(spindle_SOURCES) +DIST_SOURCES = $(libconfigparse_a_SOURCES) $(libspindlefe_la_SOURCES) \ $(libspindlefe_static_la_SOURCES) $(am__spindle_SOURCES_DIST) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ @@ -491,6 +513,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ lib_LTLIBRARIES = libspindlefe.la noinst_LTLIBRARIES = libspindlefe_static.la +noinst_LIBRARIES = libconfigparse.a include_HEADERS = $(top_srcdir)/../include/spindle_launch.h AM_CPPFLAGS = -I$(top_srcdir)/../logging CORE_SOURCES = spindle_fe.cc parseargs.cc config_parser.cc config_mgr.cc parse_preload.cc $(top_srcdir)/../utils/pathfn.c $(top_srcdir)/../utils/keyfile.c $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/rshlaunch.c @@ -515,6 +538,14 @@ spindle_LDADD = $(CORE_LDADD) \ $(top_builddir)/hostbin/libhostbin.la $(am__append_3) \ $(am__append_6) @BLD_FLUXPLUGIN_TRUE@spindle_LDFLAGS = $(fluxplugin-ldflags) -Wl,--allow-multiple-definition + +# Private static library for config parsing (used by spindle-config-helper) +libconfigparse_a_SOURCES = config_mgr.cc config_parser.cc parseargs.cc \ + $(top_srcdir)/../utils/pathfn.c \ + $(top_srcdir)/../utils/keyfile.c \ + $(top_srcdir)/../utils/parseloc.c + +libconfigparse_a_CPPFLAGS = $(CORE_CPPFLAGS) all: all-am .SUFFIXES: @@ -598,6 +629,9 @@ clean-binPROGRAMS: echo " rm -f" $$list; \ rm -f $$list +clean-noinstLIBRARIES: + -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES) + install-libLTLIBRARIES: $(lib_LTLIBRARIES) @$(NORMAL_INSTALL) @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ @@ -649,6 +683,20 @@ $(top_builddir)/../utils/$(am__dirstamp): $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp): @$(MKDIR_P) $(top_builddir)/../utils/$(DEPDIR) @: > $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/../utils/libconfigparse_a-pathfn.$(OBJEXT): \ + $(top_builddir)/../utils/$(am__dirstamp) \ + $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/../utils/libconfigparse_a-keyfile.$(OBJEXT): \ + $(top_builddir)/../utils/$(am__dirstamp) \ + $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) +$(top_builddir)/../utils/libconfigparse_a-parseloc.$(OBJEXT): \ + $(top_builddir)/../utils/$(am__dirstamp) \ + $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) + +libconfigparse.a: $(libconfigparse_a_OBJECTS) $(libconfigparse_a_DEPENDENCIES) $(EXTRA_libconfigparse_a_DEPENDENCIES) + $(AM_V_at)-rm -f libconfigparse.a + $(AM_V_AR)$(libconfigparse_a_AR) libconfigparse.a $(libconfigparse_a_OBJECTS) $(libconfigparse_a_LIBADD) + $(AM_V_at)$(RANLIB) libconfigparse.a $(top_builddir)/../utils/libspindlefe_la-pathfn.lo: \ $(top_builddir)/../utils/$(am__dirstamp) \ $(top_builddir)/../utils/$(DEPDIR)/$(am__dirstamp) @@ -704,6 +752,9 @@ mostlyclean-compile: distclean-compile: -rm -f *.tab.c +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-parseloc.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Plo@am__quote@ # am--include-marker @@ -716,6 +767,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/spindle-parseloc.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/spindle-pathfn.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@$(top_builddir)/../utils/$(DEPDIR)/spindle-rshlaunch.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libconfigparse_a-config_mgr.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libconfigparse_a-config_parser.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libconfigparse_a-parseargs.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libspindlefe_la-config_mgr.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libspindlefe_la-config_parser.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libspindlefe_la-parse_preload.Plo@am__quote@ # am--include-marker @@ -771,6 +825,48 @@ am--depfiles: $(am__depfiles_remade) @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< +$(top_builddir)/../utils/libconfigparse_a-pathfn.o: $(top_builddir)/../utils/pathfn.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libconfigparse_a-pathfn.o -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Tpo -c -o $(top_builddir)/../utils/libconfigparse_a-pathfn.o `test -f '$(top_builddir)/../utils/pathfn.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/pathfn.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Tpo $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$(top_builddir)/../utils/pathfn.c' object='$(top_builddir)/../utils/libconfigparse_a-pathfn.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o $(top_builddir)/../utils/libconfigparse_a-pathfn.o `test -f '$(top_builddir)/../utils/pathfn.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/pathfn.c + +$(top_builddir)/../utils/libconfigparse_a-pathfn.obj: $(top_builddir)/../utils/pathfn.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libconfigparse_a-pathfn.obj -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Tpo -c -o $(top_builddir)/../utils/libconfigparse_a-pathfn.obj `if test -f '$(top_builddir)/../utils/pathfn.c'; then $(CYGPATH_W) '$(top_builddir)/../utils/pathfn.c'; else $(CYGPATH_W) '$(srcdir)/$(top_builddir)/../utils/pathfn.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Tpo $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$(top_builddir)/../utils/pathfn.c' object='$(top_builddir)/../utils/libconfigparse_a-pathfn.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o $(top_builddir)/../utils/libconfigparse_a-pathfn.obj `if test -f '$(top_builddir)/../utils/pathfn.c'; then $(CYGPATH_W) '$(top_builddir)/../utils/pathfn.c'; else $(CYGPATH_W) '$(srcdir)/$(top_builddir)/../utils/pathfn.c'; fi` + +$(top_builddir)/../utils/libconfigparse_a-keyfile.o: $(top_builddir)/../utils/keyfile.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libconfigparse_a-keyfile.o -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Tpo -c -o $(top_builddir)/../utils/libconfigparse_a-keyfile.o `test -f '$(top_builddir)/../utils/keyfile.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/keyfile.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Tpo $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$(top_builddir)/../utils/keyfile.c' object='$(top_builddir)/../utils/libconfigparse_a-keyfile.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o $(top_builddir)/../utils/libconfigparse_a-keyfile.o `test -f '$(top_builddir)/../utils/keyfile.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/keyfile.c + +$(top_builddir)/../utils/libconfigparse_a-keyfile.obj: $(top_builddir)/../utils/keyfile.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libconfigparse_a-keyfile.obj -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Tpo -c -o $(top_builddir)/../utils/libconfigparse_a-keyfile.obj `if test -f '$(top_builddir)/../utils/keyfile.c'; then $(CYGPATH_W) '$(top_builddir)/../utils/keyfile.c'; else $(CYGPATH_W) '$(srcdir)/$(top_builddir)/../utils/keyfile.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Tpo $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$(top_builddir)/../utils/keyfile.c' object='$(top_builddir)/../utils/libconfigparse_a-keyfile.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o $(top_builddir)/../utils/libconfigparse_a-keyfile.obj `if test -f '$(top_builddir)/../utils/keyfile.c'; then $(CYGPATH_W) '$(top_builddir)/../utils/keyfile.c'; else $(CYGPATH_W) '$(srcdir)/$(top_builddir)/../utils/keyfile.c'; fi` + +$(top_builddir)/../utils/libconfigparse_a-parseloc.o: $(top_builddir)/../utils/parseloc.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libconfigparse_a-parseloc.o -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Tpo -c -o $(top_builddir)/../utils/libconfigparse_a-parseloc.o `test -f '$(top_builddir)/../utils/parseloc.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/parseloc.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Tpo $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$(top_builddir)/../utils/parseloc.c' object='$(top_builddir)/../utils/libconfigparse_a-parseloc.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o $(top_builddir)/../utils/libconfigparse_a-parseloc.o `test -f '$(top_builddir)/../utils/parseloc.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/parseloc.c + +$(top_builddir)/../utils/libconfigparse_a-parseloc.obj: $(top_builddir)/../utils/parseloc.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libconfigparse_a-parseloc.obj -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Tpo -c -o $(top_builddir)/../utils/libconfigparse_a-parseloc.obj `if test -f '$(top_builddir)/../utils/parseloc.c'; then $(CYGPATH_W) '$(top_builddir)/../utils/parseloc.c'; else $(CYGPATH_W) '$(srcdir)/$(top_builddir)/../utils/parseloc.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Tpo $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$(top_builddir)/../utils/parseloc.c' object='$(top_builddir)/../utils/libconfigparse_a-parseloc.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o $(top_builddir)/../utils/libconfigparse_a-parseloc.obj `if test -f '$(top_builddir)/../utils/parseloc.c'; then $(CYGPATH_W) '$(top_builddir)/../utils/parseloc.c'; else $(CYGPATH_W) '$(srcdir)/$(top_builddir)/../utils/parseloc.c'; fi` + $(top_builddir)/../utils/libspindlefe_la-pathfn.lo: $(top_builddir)/../utils/pathfn.c @am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libspindlefe_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT $(top_builddir)/../utils/libspindlefe_la-pathfn.lo -MD -MP -MF $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Tpo -c -o $(top_builddir)/../utils/libspindlefe_la-pathfn.lo `test -f '$(top_builddir)/../utils/pathfn.c' || echo '$(srcdir)/'`$(top_builddir)/../utils/pathfn.c @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Tpo $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Plo @@ -907,6 +1003,48 @@ $(top_builddir)/../utils/spindle-rshlaunch.obj: $(top_builddir)/../utils/rshlaun @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< +libconfigparse_a-config_mgr.o: config_mgr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libconfigparse_a-config_mgr.o -MD -MP -MF $(DEPDIR)/libconfigparse_a-config_mgr.Tpo -c -o libconfigparse_a-config_mgr.o `test -f 'config_mgr.cc' || echo '$(srcdir)/'`config_mgr.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libconfigparse_a-config_mgr.Tpo $(DEPDIR)/libconfigparse_a-config_mgr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='config_mgr.cc' object='libconfigparse_a-config_mgr.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libconfigparse_a-config_mgr.o `test -f 'config_mgr.cc' || echo '$(srcdir)/'`config_mgr.cc + +libconfigparse_a-config_mgr.obj: config_mgr.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libconfigparse_a-config_mgr.obj -MD -MP -MF $(DEPDIR)/libconfigparse_a-config_mgr.Tpo -c -o libconfigparse_a-config_mgr.obj `if test -f 'config_mgr.cc'; then $(CYGPATH_W) 'config_mgr.cc'; else $(CYGPATH_W) '$(srcdir)/config_mgr.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libconfigparse_a-config_mgr.Tpo $(DEPDIR)/libconfigparse_a-config_mgr.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='config_mgr.cc' object='libconfigparse_a-config_mgr.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libconfigparse_a-config_mgr.obj `if test -f 'config_mgr.cc'; then $(CYGPATH_W) 'config_mgr.cc'; else $(CYGPATH_W) '$(srcdir)/config_mgr.cc'; fi` + +libconfigparse_a-config_parser.o: config_parser.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libconfigparse_a-config_parser.o -MD -MP -MF $(DEPDIR)/libconfigparse_a-config_parser.Tpo -c -o libconfigparse_a-config_parser.o `test -f 'config_parser.cc' || echo '$(srcdir)/'`config_parser.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libconfigparse_a-config_parser.Tpo $(DEPDIR)/libconfigparse_a-config_parser.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='config_parser.cc' object='libconfigparse_a-config_parser.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libconfigparse_a-config_parser.o `test -f 'config_parser.cc' || echo '$(srcdir)/'`config_parser.cc + +libconfigparse_a-config_parser.obj: config_parser.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libconfigparse_a-config_parser.obj -MD -MP -MF $(DEPDIR)/libconfigparse_a-config_parser.Tpo -c -o libconfigparse_a-config_parser.obj `if test -f 'config_parser.cc'; then $(CYGPATH_W) 'config_parser.cc'; else $(CYGPATH_W) '$(srcdir)/config_parser.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libconfigparse_a-config_parser.Tpo $(DEPDIR)/libconfigparse_a-config_parser.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='config_parser.cc' object='libconfigparse_a-config_parser.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libconfigparse_a-config_parser.obj `if test -f 'config_parser.cc'; then $(CYGPATH_W) 'config_parser.cc'; else $(CYGPATH_W) '$(srcdir)/config_parser.cc'; fi` + +libconfigparse_a-parseargs.o: parseargs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libconfigparse_a-parseargs.o -MD -MP -MF $(DEPDIR)/libconfigparse_a-parseargs.Tpo -c -o libconfigparse_a-parseargs.o `test -f 'parseargs.cc' || echo '$(srcdir)/'`parseargs.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libconfigparse_a-parseargs.Tpo $(DEPDIR)/libconfigparse_a-parseargs.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='parseargs.cc' object='libconfigparse_a-parseargs.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libconfigparse_a-parseargs.o `test -f 'parseargs.cc' || echo '$(srcdir)/'`parseargs.cc + +libconfigparse_a-parseargs.obj: parseargs.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libconfigparse_a-parseargs.obj -MD -MP -MF $(DEPDIR)/libconfigparse_a-parseargs.Tpo -c -o libconfigparse_a-parseargs.obj `if test -f 'parseargs.cc'; then $(CYGPATH_W) 'parseargs.cc'; else $(CYGPATH_W) '$(srcdir)/parseargs.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libconfigparse_a-parseargs.Tpo $(DEPDIR)/libconfigparse_a-parseargs.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='parseargs.cc' object='libconfigparse_a-parseargs.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libconfigparse_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libconfigparse_a-parseargs.obj `if test -f 'parseargs.cc'; then $(CYGPATH_W) 'parseargs.cc'; else $(CYGPATH_W) '$(srcdir)/parseargs.cc'; fi` + libspindlefe_la-spindle_fe.lo: spindle_fe.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libspindlefe_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libspindlefe_la-spindle_fe.lo -MD -MP -MF $(DEPDIR)/libspindlefe_la-spindle_fe.Tpo -c -o libspindlefe_la-spindle_fe.lo `test -f 'spindle_fe.cc' || echo '$(srcdir)/'`spindle_fe.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libspindlefe_la-spindle_fe.Tpo $(DEPDIR)/libspindlefe_la-spindle_fe.Plo @@ -1288,7 +1426,7 @@ distdir-am: $(DISTFILES) done check-am: all-am check: check-am -all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +all-am: Makefile $(PROGRAMS) $(LIBRARIES) $(LTLIBRARIES) $(HEADERS) install-binPROGRAMS: install-libLTLIBRARIES installdirs: @@ -1330,10 +1468,14 @@ maintainer-clean-generic: clean: clean-am clean-am: clean-binPROGRAMS clean-generic clean-libLTLIBRARIES \ - clean-libtool clean-noinstLTLIBRARIES mostlyclean-am + clean-libtool clean-noinstLIBRARIES clean-noinstLTLIBRARIES \ + mostlyclean-am distclean: distclean-am - -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Po + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Po + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Po + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-parseloc.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-rshlaunch.Plo @@ -1345,6 +1487,9 @@ distclean: distclean-am -rm -f $(top_builddir)/../utils/$(DEPDIR)/spindle-parseloc.Po -rm -f $(top_builddir)/../utils/$(DEPDIR)/spindle-pathfn.Po -rm -f $(top_builddir)/../utils/$(DEPDIR)/spindle-rshlaunch.Po + -rm -f ./$(DEPDIR)/libconfigparse_a-config_mgr.Po + -rm -f ./$(DEPDIR)/libconfigparse_a-config_parser.Po + -rm -f ./$(DEPDIR)/libconfigparse_a-parseargs.Po -rm -f ./$(DEPDIR)/libspindlefe_la-config_mgr.Plo -rm -f ./$(DEPDIR)/libspindlefe_la-config_parser.Plo -rm -f ./$(DEPDIR)/libspindlefe_la-parse_preload.Plo @@ -1414,7 +1559,10 @@ install-ps-am: installcheck-am: maintainer-clean: maintainer-clean-am - -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-keyfile.Po + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-parseloc.Po + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libconfigparse_a-pathfn.Po + -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-keyfile.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-parseloc.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-pathfn.Plo -rm -f $(top_builddir)/../utils/$(DEPDIR)/libspindlefe_la-rshlaunch.Plo @@ -1426,6 +1574,9 @@ maintainer-clean: maintainer-clean-am -rm -f $(top_builddir)/../utils/$(DEPDIR)/spindle-parseloc.Po -rm -f $(top_builddir)/../utils/$(DEPDIR)/spindle-pathfn.Po -rm -f $(top_builddir)/../utils/$(DEPDIR)/spindle-rshlaunch.Po + -rm -f ./$(DEPDIR)/libconfigparse_a-config_mgr.Po + -rm -f ./$(DEPDIR)/libconfigparse_a-config_parser.Po + -rm -f ./$(DEPDIR)/libconfigparse_a-parseargs.Po -rm -f ./$(DEPDIR)/libspindlefe_la-config_mgr.Plo -rm -f ./$(DEPDIR)/libspindlefe_la-config_parser.Plo -rm -f ./$(DEPDIR)/libspindlefe_la-parse_preload.Plo @@ -1473,16 +1624,16 @@ uninstall-am: uninstall-binPROGRAMS uninstall-includeHEADERS \ .PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ clean-binPROGRAMS clean-generic clean-libLTLIBRARIES \ - clean-libtool clean-noinstLTLIBRARIES cscopelist-am ctags \ - ctags-am distclean distclean-compile distclean-generic \ - distclean-libtool distclean-tags distdir dvi dvi-am html \ - html-am info info-am install install-am install-binPROGRAMS \ - install-data install-data-am install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-includeHEADERS install-info install-info-am \ - install-libLTLIBRARIES install-man install-pdf install-pdf-am \ - install-ps install-ps-am install-strip installcheck \ - installcheck-am installdirs maintainer-clean \ + clean-libtool clean-noinstLIBRARIES clean-noinstLTLIBRARIES \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-binPROGRAMS install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-includeHEADERS install-info \ + install-info-am install-libLTLIBRARIES install-man install-pdf \ + install-pdf-am install-ps install-ps-am install-strip \ + installcheck installcheck-am installdirs maintainer-clean \ maintainer-clean-generic mostlyclean mostlyclean-compile \ mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ tags tags-am uninstall uninstall-am uninstall-binPROGRAMS \ From 44220f3c1d9aa7c3cf12ae3eea55eb6b553dacd9 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 22 Oct 2026 13:37:18 -0700 Subject: [PATCH 03/18] config_helper Milestone 2 Add filesystem analysis and path validation to spindle-config-helper (milestone 2) Enhances spindle-config-helper to analyze and validate configured paths, helping users identify problematic configurations before runtime. Features: - Analyzes each path in commpaths, cachepaths, and localprefix - Detects and warns about remote filesystems (NFS, Lustre, GPFS, etc.) - Validates environment variable expansion - Detects undefined variables ($BADVAR) - Detects empty variables creating double-slashes (/var/$EMPTY/dir) - Shows both original and expanded paths - Checks parent directory permissions for non-existent paths - Resolves relative paths and symlinks to show real locations - Hardcodes kernel filesystem paths (/proc, /sys, /dev) as local - Treats overlay filesystems as local (container backing storage) Output format provides per-path analysis with status (OK/WARNING), original path, resolved path, filesystem type, and specific warnings: commpaths analysis: OK: $TMPDIR -> /var/tmp/rountree (tmpfs, 250GB) WARNING: /var/$EMPTY/dir -> /var//dir Path contains empty segment (variable may be empty) Parent directory (/var) is not writable WARNING: /nfs/cache -> /nfs/cache (nfs4) Path is on REMOTE filesystem (nfs4) This detailed feedback helps users and administrators diagnose configuration issues by showing exactly what each path expands to, where it resolves, and why it might not work. Technical details: - Uses wordexp(3) with WRDE_UNDEF for variable expansion - Uses realpath(3) for symlink/relative path resolution - Parses /proc/mounts with 1GB minimum space threshold - Checks write permission on nearest existing parent directory - Filters duplicate mounts and system/virtual filesystems Co-Authored-By: Claude --- src/config_helper/spindle-config-helper.cc | 441 +++++++++++++++++++++ 1 file changed, 441 insertions(+) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index c1237bca..e3cd91d8 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -17,9 +17,444 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "config.h" #include "config_mgr.h" #include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include +#include + +struct MountPoint { + std::string path; + std::string fstype; + bool is_remote; + uint64_t avail_gb; + + MountPoint(const std::string& p, const std::string& fs) + : path(p), fstype(fs), is_remote(false), avail_gb(0) {} +}; + +static bool is_remote_filesystem(const std::string& fstype) +{ + static const char* remote_types[] = { + "nfs", "nfs4", "lustre", "gpfs", "panfs", + "ceph", "glusterfs", "afs", "smbfs", "cifs", + nullptr + }; + + for (int i = 0; remote_types[i]; i++) { + if (fstype == remote_types[i] || + fstype.find(std::string("fuse.") + remote_types[i]) == 0) { + return true; + } + } + return false; +} + +static bool is_kernel_filesystem(const std::string& path) +{ + // Hardcode known kernel filesystem paths + // Note: /dev/shm is NOT a kernel filesystem, it's tmpfs + return path == "/proc" || path == "/sys" || + (path.find("/proc/") == 0) || + (path.find("/sys/") == 0) || + (path == "/dev" || (path.find("/dev/") == 0 && path != "/dev/shm")); +} + +static bool should_ignore_mount(const std::string& path, const std::string& fstype) +{ + // Ignore virtual/system filesystems + if (fstype == "proc" || fstype == "sysfs" || fstype == "devtmpfs" || + fstype == "devpts" || fstype == "securityfs" || fstype == "pstore" || + fstype == "autofs" || fstype == "mqueue" || fstype == "hugetlbfs" || + fstype.find("cgroup") == 0 || fstype.find("bpf") == 0) { + return true; + } + + // Ignore system tmpfs mounts (but NOT /dev/shm which is useful) + if (fstype == "tmpfs" && (path == "/run" || path == "/sys/fs/cgroup" || + path.find("/run/") == 0 || path.find("/sys/") == 0 || + (path.find("/dev/") == 0 && path != "/dev/shm"))) { + return true; + } + + return false; +} + +static uint64_t get_available_gb(const std::string& path) +{ + struct statvfs stat; + if (statvfs(path.c_str(), &stat) != 0) { + return 0; + } + + uint64_t avail_bytes = (uint64_t)stat.f_bavail * stat.f_frsize; + return avail_bytes / (1024ULL * 1024 * 1024); +} + +static std::vector parse_mounts() +{ + std::vector mounts; + std::ifstream mounts_file("/proc/mounts"); + std::string line; + std::set seen_paths; + + while (std::getline(mounts_file, line)) { + std::istringstream iss(line); + std::string device, path, fstype, options; + + if (!(iss >> device >> path >> fstype >> options)) { + continue; + } + + if (should_ignore_mount(path, fstype)) { + continue; + } + + // Skip duplicate paths + if (seen_paths.count(path)) { + continue; + } + seen_paths.insert(path); + + MountPoint mp(path, fstype); + + // Overlay filesystems are typically local (container backing storage) + if (fstype == "overlay") { + mp.is_remote = false; + } else { + mp.is_remote = is_remote_filesystem(fstype); + } + + if (!mp.is_remote) { + mp.avail_gb = get_available_gb(path); + + // Skip local mounts with < 1GB available + if (mp.avail_gb < 1) { + continue; + } + + // Check if writable + if (access(path.c_str(), W_OK) != 0) { + continue; + } + } + + mounts.push_back(mp); + } + + // Sort by path length (longest first) for prefix matching + std::sort(mounts.begin(), mounts.end(), + [](const MountPoint& a, const MountPoint& b) { + return a.path.length() > b.path.length(); + }); + + return mounts; +} + +enum ExpandResult { + EXPAND_OK, + EXPAND_UNDEFINED_VAR, + EXPAND_EMPTY, + EXPAND_ERROR +}; + +struct ExpandedPath { + std::string expanded; + ExpandResult result; +}; + +static ExpandedPath expand_path(const std::string& path) +{ + ExpandedPath ep; + + if (path.empty()) { + ep.expanded = path; + ep.result = EXPAND_EMPTY; + return ep; + } + + // Use wordexp for shell expansion (handles $VAR, ~, etc.) + wordexp_t exp_result; + int ret = wordexp(path.c_str(), &exp_result, WRDE_NOCMD | WRDE_UNDEF); + + if (ret == WRDE_BADVAL) { + // Undefined variable + ep.expanded = ""; + ep.result = EXPAND_UNDEFINED_VAR; + return ep; + } else if (ret != 0) { + // Other expansion error + ep.expanded = ""; + ep.result = EXPAND_ERROR; + return ep; + } + + if (exp_result.we_wordc > 0 && exp_result.we_wordv[0] && exp_result.we_wordv[0][0] != '\0') { + ep.expanded = exp_result.we_wordv[0]; + ep.result = EXPAND_OK; + } else { + // Empty expansion + ep.expanded = ""; + ep.result = EXPAND_EMPTY; + } + + wordfree(&exp_result); + return ep; +} + +static std::string resolve_real_path(const std::string& path) +{ + if (path.empty()) { + return path; + } + + char resolved[PATH_MAX]; + if (realpath(path.c_str(), resolved) != nullptr) { + return std::string(resolved); + } + + // If realpath fails, try to handle relative paths manually + if (path[0] != '/') { + char cwd[PATH_MAX]; + if (getcwd(cwd, sizeof(cwd))) { + std::string abs = std::string(cwd) + "/" + path; + if (realpath(abs.c_str(), resolved) != nullptr) { + return std::string(resolved); + } + return abs; // Return constructed absolute path even if realpath fails + } + } + + return path; // Return original if we can't resolve +} + +static const MountPoint* find_mount_for_path(const std::string& resolved_path, + const std::vector& mounts) +{ + if (resolved_path.empty()) { + return nullptr; + } + + // Find longest matching mount point + for (const auto& mp : mounts) { + if (resolved_path == mp.path || + (resolved_path.find(mp.path + "/") == 0)) { + return ∓ + } + } + + return nullptr; +} + +struct PathAnalysis { + std::string original; + std::string expanded; + std::string resolved; + bool is_ok; + std::vector warnings; + const MountPoint* mount; + + PathAnalysis(const std::string& orig) : original(orig), is_ok(false), mount(nullptr) {} +}; + +static std::string find_writable_parent(const std::string& path) +{ + // Find the first existing parent directory + std::string test_path = path; + + while (!test_path.empty() && test_path != "/") { + if (access(test_path.c_str(), F_OK) == 0) { + // Path exists, check if writable + return test_path; + } + + // Move to parent + size_t last_slash = test_path.rfind('/'); + if (last_slash == std::string::npos) { + break; + } + test_path = test_path.substr(0, last_slash); + if (test_path.empty()) { + test_path = "/"; + } + } + + return "/"; +} + +static PathAnalysis analyze_path(const std::string& path, const std::vector& mounts) +{ + PathAnalysis analysis(path); + + if (path.empty()) { + analysis.warnings.push_back("Path is empty"); + return analysis; + } + + // Expand variables + ExpandedPath ep = expand_path(path); + analysis.expanded = ep.expanded; + + if (ep.result == EXPAND_UNDEFINED_VAR) { + analysis.warnings.push_back("Variable is not set"); + return analysis; + } else if (ep.result == EXPAND_EMPTY) { + analysis.warnings.push_back("Variable expands to empty string"); + return analysis; + } else if (ep.result == EXPAND_ERROR) { + analysis.warnings.push_back("Cannot expand path"); + return analysis; + } + + // Check for empty segments (e.g., /var// from /var/$EMPTY/) + if (analysis.expanded.find("//") != std::string::npos) { + analysis.warnings.push_back("Path contains empty segment (variable may be empty)"); + } + + // Resolve to real path + analysis.resolved = resolve_real_path(ep.expanded); + + // Check for kernel filesystems first (these are always OK) + if (is_kernel_filesystem(analysis.resolved)) { + analysis.is_ok = true; + return analysis; + } + + // Check if path exists + struct stat st; + if (stat(analysis.resolved.c_str(), &st) != 0) { + analysis.warnings.push_back("Path does not exist (will be created by Spindle)"); + + // Check if parent is writable + std::string parent = find_writable_parent(analysis.resolved); + if (access(parent.c_str(), W_OK) != 0) { + analysis.warnings.push_back("Parent directory (" + parent + ") is not writable"); + return analysis; + } + } + + // Find mount point + analysis.mount = find_mount_for_path(analysis.resolved, mounts); + + if (!analysis.mount) { + analysis.warnings.push_back("Cannot determine filesystem type"); + return analysis; + } + + if (analysis.mount->is_remote) { + analysis.warnings.push_back("Path is on REMOTE filesystem (" + analysis.mount->fstype + ")"); + return analysis; + } + + // All checks passed + analysis.is_ok = true; + return analysis; +} + +static void print_filesystem_recommendations(const std::vector& mounts, + const ConfigMap& confmap) +{ + std::cout << "# Spindle filesystem recommendations:" << std::endl; + + // Show local writable mounts + std::vector local_mounts; + for (const auto& mp : mounts) { + if (!mp.is_remote) { + local_mounts.push_back(&mp); + } + } + + if (!local_mounts.empty()) { + std::cout << "# Local: "; + bool first = true; + for (const auto* mp : local_mounts) { + if (!first) std::cout << ", "; + std::cout << mp->path << " (" << mp->fstype << ", " + << mp->avail_gb << "GB)"; + first = false; + } + std::cout << std::endl; + } else { + std::cout << "# WARNING: No suitable local filesystems found!" << std::endl; + } + + std::cout << "#" << std::endl; + + // Analyze each configured option + auto analyze_option = [&](SpindleConfigID config_id, const char* name) { + if (!confmap.isSet(config_id)) { + return; + } + + auto result = confmap.getValueString(config_id); + if (!result.first || result.second.empty()) { + return; + } + + std::string paths = result.second; + std::vector analyses; + + // Split on colons for multi-path options + size_t pos = 0; + while (pos < paths.length()) { + size_t end = paths.find(':', pos); + if (end == std::string::npos) { + end = paths.length(); + } + + std::string path = paths.substr(pos, end - pos); + if (!path.empty()) { + analyses.push_back(analyze_path(path, mounts)); + } + + pos = end + 1; + } + + if (analyses.empty()) { + return; + } + + std::cout << "# " << name << " analysis:" << std::endl; + + for (const auto& analysis : analyses) { + const char* status = analysis.is_ok ? "OK" : "WARNING"; + std::cout << "# " << status << ": " << analysis.original; + + if (!analysis.resolved.empty()) { + std::cout << " -> " << analysis.resolved; + if (analysis.mount && !analysis.mount->is_remote) { + std::cout << " (" << analysis.mount->fstype << ", " + << analysis.mount->avail_gb << "GB)"; + } else if (analysis.mount && analysis.mount->is_remote) { + std::cout << " (" << analysis.mount->fstype << ")"; + } + } else if (!analysis.expanded.empty()) { + std::cout << " -> " << analysis.expanded; + } else { + std::cout << " -> (empty)"; + } + + std::cout << std::endl; + + for (const auto& warning : analysis.warnings) { + std::cout << "# " << warning << std::endl; + } + } + + std::cout << "#" << std::endl; + }; + + analyze_option(confCommPaths, "commpaths"); + analyze_option(confCachePaths, "cachepaths"); + analyze_option(confLocalPrefix, "localprefix"); +} static void print_version() { @@ -158,6 +593,12 @@ int main(int argc, char *argv[]) delete[] modified_argv; } + // Parse and analyze mount points + std::vector mounts = parse_mounts(); + + // Print filesystem recommendations and warnings + print_filesystem_recommendations(mounts, confmap); + // Output configuration in spindle.conf format const std::map &options = getOptionsByID(); From 108ceff14db9db5d509af90203975ef1aa0ca87d Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 22 Oct 2026 14:05:56 -0700 Subject: [PATCH 04/18] config_helper Adds testsuite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a lightweight integration test suite to validate spindle-config-helper functionality and catch regressions. Test coverage (10 tests, all passing): - Basic execution (smoke test) - Command-line flags (--help, --version) - Local filesystem detection (/dev/shm, /tmp) - Variable handling (undefined, empty) - Multiple colon-separated paths - Kernel filesystem recognition (/proc, /sys, /dev) - Parent directory permission checking - Relative path resolution Structure: testsuite/config_helper/ ├── run_tests.sh Test runner script ├── README.md Documentation ├── sample_mounts/ Example /proc/mounts files │ ├── simple.mounts Basic NFS + local │ ├── hpc.mounts Lustre + GPFS + local │ └── container.mounts Overlay + tmpfs └── expected/ (Reserved for golden files) Design philosophy: - Simple integration tests that run the actual binary - Use real filesystem operations where possible - No heavy mocking - trust implementation, catch regressions - Fast execution (~1 second) - Easy to extend with new test functions Sample mount files are provided for future enhancements if we add deterministic testing against various filesystem configurations. Usage: cd testsuite/config_helper ./run_tests.sh /path/to/spindle-config-helper Exit codes: 0 (all passed), 1 (one or more failed) Co-Authored-By: Claude --- testsuite/config_helper/README.md | 58 ++++++++ testsuite/config_helper/run_tests.sh | 132 ++++++++++++++++++ .../sample_mounts/container.mounts | 6 + .../config_helper/sample_mounts/hpc.mounts | 8 ++ .../config_helper/sample_mounts/simple.mounts | 7 + 5 files changed, 211 insertions(+) create mode 100644 testsuite/config_helper/README.md create mode 100755 testsuite/config_helper/run_tests.sh create mode 100644 testsuite/config_helper/sample_mounts/container.mounts create mode 100644 testsuite/config_helper/sample_mounts/hpc.mounts create mode 100644 testsuite/config_helper/sample_mounts/simple.mounts diff --git a/testsuite/config_helper/README.md b/testsuite/config_helper/README.md new file mode 100644 index 00000000..b13d4580 --- /dev/null +++ b/testsuite/config_helper/README.md @@ -0,0 +1,58 @@ +# spindle-config-helper Test Suite + +Simple integration tests for the `spindle-config-helper` tool. + +## Running Tests + +```bash +# From the build directory +cd testsuite/config_helper +./run_tests.sh /path/to/spindle-config-helper + +# Or let it find the default location +./run_tests.sh +``` + +## Test Coverage + +1. **test_01_smoke** - Basic execution (does it run without crashing?) +2. **test_02_help** - Help flag works +3. **test_03_version** - Version flag works +4. **test_04_local_fs** - Detects local filesystems correctly (/dev/shm, /tmp) +5. **test_05_undefined_var** - Warns about undefined variables +6. **test_06_empty_var** - Detects empty variables creating // in paths +7. **test_07_multiple_paths** - Handles colon-separated path lists +8. **test_08_kernel_fs** - Recognizes kernel filesystems (/proc, /sys, /dev) +9. **test_09_permissions** - Checks parent directory permissions +10. **test_10_relative_path** - Resolves relative paths correctly + +## Sample Mount Files + +The `sample_mounts/` directory contains example `/proc/mounts` files for testing: + +- **simple.mounts** - Basic system with NFS and local storage +- **hpc.mounts** - HPC system with Lustre, GPFS, and local storage +- **container.mounts** - Container environment with overlay filesystem + +These can be used for future mocking/testing if we implement a `--test-mounts` flag. + +## Exit Codes + +- `0` - All tests passed +- `1` - One or more tests failed + +## Adding New Tests + +1. Add a `test_XX_description()` function to `run_tests.sh` +2. Add `run_test "XX_description" test_XX_description` to the test list +3. Test should return 0 on success, non-zero on failure +4. Use `grep -q` for output validation to keep tests simple + +## Design Philosophy + +These are **integration tests**, not unit tests: +- Run the actual tool binary +- Use real filesystem operations where possible +- Keep tests simple and maintainable +- Focus on catching regressions, not exhaustive coverage +- Avoid heavy mocking - trust the implementation for complex logic diff --git a/testsuite/config_helper/run_tests.sh b/testsuite/config_helper/run_tests.sh new file mode 100755 index 00000000..fad1dbf4 --- /dev/null +++ b/testsuite/config_helper/run_tests.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Test suite for spindle-config-helper +# Usage: ./run_tests.sh [path-to-spindle-config-helper] + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HELPER="${1:-../../build/config_helper-serial/src/config_helper/spindle-config-helper}" + +if [[ ! -x "$HELPER" ]]; then + echo "Error: spindle-config-helper not found or not executable: $HELPER" + echo "Usage: $0 [path-to-spindle-config-helper]" + exit 1 +fi + +TESTS_PASSED=0 +TESTS_FAILED=0 +FAILED_TESTS="" + +# Test helper function +run_test() { + local test_name="$1" + local test_func="$2" + + echo -n "Running test: $test_name ... " + + if $test_func 2>&1 >/dev/null; then + echo "PASSED" + ((TESTS_PASSED++)) + return 0 + else + echo "FAILED" + ((TESTS_FAILED++)) + FAILED_TESTS="$FAILED_TESTS\n - $test_name" + return 1 + fi +} + +# Test 1: Basic smoke test - does it run? +test_01_smoke() { + $HELPER --commpaths=/tmp --cachepaths=/tmp >/dev/null 2>&1 +} + +# Test 2: Help flag works +test_02_help() { + $HELPER --help 2>&1 | grep -q "Usage:" +} + +# Test 3: Version flag works +test_03_version() { + $HELPER --version 2>&1 | grep -q "spindle-config-helper version" +} + +# Test 4: Local filesystem detection +test_04_local_fs() { + output=$($HELPER --commpaths=/dev/shm --cachepaths=/tmp 2>&1) + echo "$output" | grep -q "OK: /dev/shm" && \ + echo "$output" | grep -q "OK: /tmp" +} + +# Test 5: Undefined variable detection +test_05_undefined_var() { + unset SPINDLE_TEST_VAR + output=$($HELPER --commpaths='$SPINDLE_TEST_VAR' 2>&1) + echo "$output" | grep -q "Variable is not set" +} + +# Test 6: Empty variable detection +test_06_empty_var() { + export SPINDLE_TEST_VAR="" + output=$($HELPER --commpaths='/tmp/$SPINDLE_TEST_VAR/dir' 2>&1) + echo "$output" | grep -q "empty segment" +} + +# Test 7: Multiple paths with colon separator +test_07_multiple_paths() { + output=$($HELPER --commpaths=/tmp:/dev/shm 2>&1) + echo "$output" | grep -q "OK: /tmp" && \ + echo "$output" | grep -q "OK: /dev/shm" +} + +# Test 8: Kernel filesystem handling +test_08_kernel_fs() { + output=$($HELPER --local-prefix=/proc:/sys:/dev 2>&1) + echo "$output" | grep -q "OK: /proc" && \ + echo "$output" | grep -q "OK: /sys" && \ + echo "$output" | grep -q "OK: /dev" +} + +# Test 9: Permission check for non-existent paths +test_09_permissions() { + # /tmp should be writable + output=$($HELPER --commpaths=/tmp/spindle_test_nonexistent_dir 2>&1) + echo "$output" | grep -q "OK: /tmp/spindle_test_nonexistent_dir" && \ + echo "$output" | grep -q "will be created by Spindle" +} + +# Test 10: Relative path resolution +test_10_relative_path() { + cd /tmp + output=$($HELPER --commpaths=./relative_test 2>&1) + echo "$output" | grep -q "/tmp/relative_test" + cd "$SCRIPT_DIR" +} + +# Run all tests +echo "=========================================" +echo "spindle-config-helper Test Suite" +echo "=========================================" +echo "Testing: $HELPER" +echo "" + +run_test "01_smoke_test" test_01_smoke +run_test "02_help_flag" test_02_help +run_test "03_version_flag" test_03_version +run_test "04_local_filesystem_detection" test_04_local_fs +run_test "05_undefined_variable" test_05_undefined_var +run_test "06_empty_variable" test_06_empty_var +run_test "07_multiple_paths" test_07_multiple_paths +run_test "08_kernel_filesystems" test_08_kernel_fs +run_test "09_permission_check" test_09_permissions +run_test "10_relative_path_resolution" test_10_relative_path + +echo "" +echo "=========================================" +echo "Results: $TESTS_PASSED passed, $TESTS_FAILED failed" +echo "=========================================" + +if [[ $TESTS_FAILED -gt 0 ]]; then + echo -e "Failed tests:$FAILED_TESTS" + exit 1 +fi + +exit 0 diff --git a/testsuite/config_helper/sample_mounts/container.mounts b/testsuite/config_helper/sample_mounts/container.mounts new file mode 100644 index 00000000..57e25043 --- /dev/null +++ b/testsuite/config_helper/sample_mounts/container.mounts @@ -0,0 +1,6 @@ +overlay / overlay rw,relatime,lowerdir=/sysroot,upperdir=/overlay/rw,workdir=/overlay/work 0 0 +tmpfs /dev/shm tmpfs rw,nosuid,nodev,size=8G 0 0 +tmpfs /tmp tmpfs rw,nosuid,nodev,size=16G 0 0 +proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 +sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 +devtmpfs /dev devtmpfs rw,relatime,size=4G 0 0 diff --git a/testsuite/config_helper/sample_mounts/hpc.mounts b/testsuite/config_helper/sample_mounts/hpc.mounts new file mode 100644 index 00000000..37f08a3d --- /dev/null +++ b/testsuite/config_helper/sample_mounts/hpc.mounts @@ -0,0 +1,8 @@ +tmpfs /dev/shm tmpfs rw,nosuid,nodev,size=128G 0 0 +tmpfs /tmp tmpfs rw,nosuid,nodev,size=100G 0 0 +lustre-mdt:/scratch /scratch lustre rw,lazystatfs 0 0 +gpfs-server:/gpfs/project /gpfs/project gpfs rw,relatime 0 0 +nfs-server:/home /home nfs4 rw,relatime,vers=4.2 0 0 +/dev/nvme0n1 /local ext4 rw,relatime 0 0 +proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 +sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 diff --git a/testsuite/config_helper/sample_mounts/simple.mounts b/testsuite/config_helper/sample_mounts/simple.mounts new file mode 100644 index 00000000..0d23a0c4 --- /dev/null +++ b/testsuite/config_helper/sample_mounts/simple.mounts @@ -0,0 +1,7 @@ +tmpfs /dev/shm tmpfs rw,nosuid,nodev,size=64G 0 0 +tmpfs /tmp tmpfs rw,nosuid,nodev,size=32G 0 0 +nfs-server:/exports/home /home nfs4 rw,relatime,vers=4.1 0 0 +/dev/sda1 /data ext4 rw,relatime 0 0 +proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 +sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 +devtmpfs /dev devtmpfs rw,nosuid,size=16G,mode=755 0 0 From 6812f07cafebb73e3485d2bf83cef44bf697c439 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 22 Oct 2026 14:28:03 -0700 Subject: [PATCH 05/18] [config_helper] Adds pythonprefix handling. Commit Message 1: Python Prefix Validation Add pythonprefix validation to spindle-config-helper Validates that pythonprefix paths point to valid Python installations. Background: Spindle uses pythonprefix for prefix matching on file paths. The default is "/usr/lib/python:/usr/lib64/python" which matches paths like: /usr/lib/python3.11/site-packages/numpy/__init__.py /usr/lib/python3.11/lib-dynload/_ctypes.so The prefix should be the Python installation root (e.g., /usr/tce or /opt/cray/pe/python/3.12.12), NOT the lib subdirectory. Validation rules: - Path must exist (no point in intercepting files that don't exist) - No undefined variables (catches $PYTHON_HOME when unset) - Must contain bin/python or bin/python3 (validates it's a Python install) Example output: pythonprefix analysis: OK: /usr/tce -> /usr/tce WARNING: /tmp -> /tmp No bin/python or bin/python3 found (not a Python installation?) WARNING: $PYTHON_HOME -> (empty) Variable is not set This helps users catch common mistakes like: - Setting prefix to a non-existent path - Using /usr/lib/python/3.11 instead of /usr/lib/python - Pointing to a non-Python directory - Relying on undefined environment variables Co-Authored-By: Claude Commit Message 2: Test Suite Updates Add pythonprefix tests to spindle-config-helper test suite Adds 3 new tests for pythonprefix validation (13 tests total, all passing): 11. test_11_pythonprefix_valid - Valid Python installation Uses /usr/tce if available, skips gracefully otherwise 12. test_12_pythonprefix_invalid - Path without Python binaries Tests /tmp which exists but has no bin/python 13. test_13_pythonprefix_undefined_var - Undefined variable detection Tests $PYTHON_HOME when unset Also fixes test_09_permissions which broke when pythonprefix validation changed path-doesn't-exist behavior from OK to WARNING. All tests are portable: - Test 11 skips on systems without /usr/tce - Tests 12 and 13 work on any system - No searching, no external dependencies - Fast execution Co-Authored-By: Claude --- src/config_helper/spindle-config-helper.cc | 71 ++++++++++++++++------ testsuite/config_helper/README.md | 3 + testsuite/config_helper/run_tests.sh | 30 ++++++++- 3 files changed, 82 insertions(+), 22 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index e3cd91d8..3ac4ec1e 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -289,7 +289,8 @@ static std::string find_writable_parent(const std::string& path) return "/"; } -static PathAnalysis analyze_path(const std::string& path, const std::vector& mounts) +static PathAnalysis analyze_path(const std::string& path, const std::vector& mounts, + bool check_for_python = false) { PathAnalysis analysis(path); @@ -328,33 +329,62 @@ static PathAnalysis analyze_path(const std::string& path, const std::vectoris_remote) { - analysis.warnings.push_back("Path is on REMOTE filesystem (" + analysis.mount->fstype + ")"); - return analysis; + // Find mount point (only for cache/comm paths, not python prefix) + if (!check_for_python) { + analysis.mount = find_mount_for_path(analysis.resolved, mounts); + + if (!analysis.mount) { + analysis.warnings.push_back("Cannot determine filesystem type"); + return analysis; + } + + if (analysis.mount->is_remote) { + analysis.warnings.push_back("Path is on REMOTE filesystem (" + analysis.mount->fstype + ")"); + return analysis; + } + } + + // All checks passed (or acceptable warnings) + if (analysis.warnings.empty()) { + analysis.is_ok = true; } - // All checks passed - analysis.is_ok = true; return analysis; } @@ -388,7 +418,7 @@ static void print_filesystem_recommendations(const std::vector& moun std::cout << "#" << std::endl; // Analyze each configured option - auto analyze_option = [&](SpindleConfigID config_id, const char* name) { + auto analyze_option = [&](SpindleConfigID config_id, const char* name, bool check_python = false) { if (!confmap.isSet(config_id)) { return; } @@ -411,7 +441,7 @@ static void print_filesystem_recommendations(const std::vector& moun std::string path = paths.substr(pos, end - pos); if (!path.empty()) { - analyses.push_back(analyze_path(path, mounts)); + analyses.push_back(analyze_path(path, mounts, check_python)); } pos = end + 1; @@ -453,6 +483,7 @@ static void print_filesystem_recommendations(const std::vector& moun analyze_option(confCommPaths, "commpaths"); analyze_option(confCachePaths, "cachepaths"); + analyze_option(confPythonPrefix, "pythonprefix", true); analyze_option(confLocalPrefix, "localprefix"); } diff --git a/testsuite/config_helper/README.md b/testsuite/config_helper/README.md index b13d4580..5df01719 100644 --- a/testsuite/config_helper/README.md +++ b/testsuite/config_helper/README.md @@ -25,6 +25,9 @@ cd testsuite/config_helper 8. **test_08_kernel_fs** - Recognizes kernel filesystems (/proc, /sys, /dev) 9. **test_09_permissions** - Checks parent directory permissions 10. **test_10_relative_path** - Resolves relative paths correctly +11. **test_11_pythonprefix_valid** - Validates Python installation (skips if /usr/tce missing) +12. **test_12_pythonprefix_invalid** - Warns when path lacks bin/python +13. **test_13_pythonprefix_undefined_var** - Detects undefined variables in pythonprefix ## Sample Mount Files diff --git a/testsuite/config_helper/run_tests.sh b/testsuite/config_helper/run_tests.sh index fad1dbf4..75befe75 100755 --- a/testsuite/config_helper/run_tests.sh +++ b/testsuite/config_helper/run_tests.sh @@ -87,9 +87,9 @@ test_08_kernel_fs() { # Test 9: Permission check for non-existent paths test_09_permissions() { - # /tmp should be writable + # /tmp should be writable - warns about non-existence but shows it can be created output=$($HELPER --commpaths=/tmp/spindle_test_nonexistent_dir 2>&1) - echo "$output" | grep -q "OK: /tmp/spindle_test_nonexistent_dir" && \ + echo "$output" | grep -q "/tmp/spindle_test_nonexistent_dir" && \ echo "$output" | grep -q "will be created by Spindle" } @@ -101,6 +101,29 @@ test_10_relative_path() { cd "$SCRIPT_DIR" } +# Test 11: pythonprefix with valid installation +test_11_pythonprefix_valid() { + # Use /usr/tce if it exists, otherwise skip + if [[ ! -x /usr/tce/bin/python3 ]]; then + return 0 # Skip on systems without /usr/tce + fi + output=$($HELPER --python-prefix=/usr/tce 2>&1) + echo "$output" | grep -q "OK: /usr/tce" +} + +# Test 12: pythonprefix without Python binaries +test_12_pythonprefix_invalid() { + output=$($HELPER --python-prefix=/tmp 2>&1) + echo "$output" | grep -q "No bin/python or bin/python3 found" +} + +# Test 13: pythonprefix with undefined variable +test_13_pythonprefix_undefined_var() { + unset PYTHON_HOME + output=$($HELPER --python-prefix='$PYTHON_HOME' 2>&1) + echo "$output" | grep -q "Variable is not set" +} + # Run all tests echo "=========================================" echo "spindle-config-helper Test Suite" @@ -118,6 +141,9 @@ run_test "07_multiple_paths" test_07_multiple_paths run_test "08_kernel_filesystems" test_08_kernel_fs run_test "09_permission_check" test_09_permissions run_test "10_relative_path_resolution" test_10_relative_path +run_test "11_pythonprefix_valid" test_11_pythonprefix_valid +run_test "12_pythonprefix_invalid" test_12_pythonprefix_invalid +run_test "13_pythonprefix_undefined_var" test_13_pythonprefix_undefined_var echo "" echo "=========================================" From f63d4c103817a2df7bf37e940a6bbc4c8e741c41 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 22 Oct 2026 17:10:00 -0700 Subject: [PATCH 06/18] [config_helper] Security validation. Implements comprehensive security validation including: - Detection of compiled security mode and current runtime setting - Mode-specific validation: * none: warns about insecure connections * keyfile: validates keydir is on shared filesystem * launchmon: warns about deprecated mode * munge: verifies daemon availability - Sandbox-aware munge testing (detects SANDBOX_ENGINE env var) - Always recommends munge when available New helper function get_sec_keydir_path() in keyfile.c exposes SEC_KEYDIR compile-time constant for keyfile validation. Test suite expanded with tests 14-17 covering security mode detection, warnings, and munge availability checks. All 17 tests passing. Co-Authored-By: Claude --- src/config_helper/spindle-config-helper.cc | 118 +++++++++++++++++++++ src/utils/keyfile.c | 9 ++ src/utils/keyfile.h | 1 + testsuite/config_helper/README.md | 4 + testsuite/config_helper/run_tests.sh | 30 ++++++ 5 files changed, 162 insertions(+) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 3ac4ec1e..71564559 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -29,6 +29,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include +#include + +extern "C" { +#include "keyfile.h" +} struct MountPoint { std::string path; @@ -492,6 +498,115 @@ static void print_version() std::cout << "spindle-config-helper version " << PACKAGE_VERSION << std::endl; } +static bool check_munge_available(bool in_sandbox) +{ + // Check if munge binary exists + if (access("/usr/bin/munge", X_OK) != 0) { + return false; + } + + // If in sandbox, we can't test the daemon + if (in_sandbox) { + return true; // Binary exists, can't test further + } + + // Try to actually use munge to verify daemon is running + pid_t pid = fork(); + if (pid == 0) { + // Child: run munge -n with stdout/stderr redirected to /dev/null + int devnull = open("/dev/null", O_WRONLY); + dup2(devnull, STDOUT_FILENO); + dup2(devnull, STDERR_FILENO); + close(devnull); + execl("/usr/bin/munge", "munge", "-n", (char*)NULL); + _exit(127); + } else if (pid > 0) { + // Parent: wait for child + int status; + waitpid(pid, &status, 0); + return WIFEXITED(status) && WEXITSTATUS(status) == 0; + } + + return false; // fork failed +} + +static void print_security_analysis(const ConfigMap& confmap, const std::vector& mounts) +{ + std::cout << "# Security configuration:" << std::endl; + + // Detect if we're in a sandbox + const char* sandbox_engine = getenv("SANDBOX_ENGINE"); + bool in_sandbox = (sandbox_engine != NULL); + + // Get the current security setting + auto sec_result = confmap.getValueString(confSecurity); + std::string current_mode = "unknown"; + if (sec_result.first) { + current_mode = sec_result.second; + } + + std::cout << "# Current mode: " << current_mode << std::endl; + + // Validate based on mode + if (current_mode == "none") { + std::cout << "# WARNING: No authentication enabled (connections are insecure)" << std::endl; + } else if (current_mode == "keyfile") { + const char* keydir = get_sec_keydir_path(); + if (keydir) { + std::cout << "# Keyfile directory: " << keydir << std::endl; + + // Check if keydir is on a shared (remote) filesystem + std::string keydir_resolved = resolve_real_path(keydir); + const MountPoint* mp = find_mount_for_path(keydir_resolved, mounts); + + if (mp) { + if (mp->is_remote) { + std::cout << "# OK: Keyfile directory is on shared filesystem (" + << mp->fstype << ")" << std::endl; + } else { + std::cout << "# WARNING: Keyfile directory is on LOCAL filesystem (" + << mp->fstype << ")" << std::endl; + std::cout << "# Keyfile must be on a filesystem shared across all nodes" << std::endl; + } + } else { + std::cout << "# WARNING: Cannot determine filesystem type for keyfile directory" << std::endl; + } + } else { + std::cout << "# Note: Keyfile mode not compiled in" << std::endl; + } + } else if (current_mode == "launchmon") { + std::cout << "# WARNING: LaunchMON security mode (deprecated/untested)" << std::endl; + } else if (current_mode == "munge") { + std::cout << "# Munge authentication enabled" << std::endl; + } + + std::cout << "#" << std::endl; + std::cout << "# Munge availability:" << std::endl; + + // Always check for munge as a recommendation + bool munge_binary_exists = (access("/usr/bin/munge", X_OK) == 0); + + if (munge_binary_exists) { + if (in_sandbox) { + std::cout << "# Munge binary found at /usr/bin/munge" << std::endl; + std::cout << "# Note: Cannot test munge daemon (running in sandbox)" << std::endl; + } else { + bool munge_works = check_munge_available(false); + if (munge_works) { + std::cout << "# Munge is available and working (recommended security option)" << std::endl; + } else { + std::cout << "# Munge binary found but daemon is not running" << std::endl; + std::cout << "# Munge provides secure authentication (recommended if available)" << std::endl; + } + } + } else { + std::cout << "# Munge not found" << std::endl; + std::cout << "# Munge provides secure authentication (recommended if available)" << std::endl; + } + + std::cout << "#" << std::endl; +} + static void print_usage(const char *progname) { std::cout << "Usage: " << progname << " [OPTIONS]\n\n" @@ -627,6 +742,9 @@ int main(int argc, char *argv[]) // Parse and analyze mount points std::vector mounts = parse_mounts(); + // Print security analysis + print_security_analysis(confmap, mounts); + // Print filesystem recommendations and warnings print_filesystem_recommendations(mounts, confmap); diff --git a/src/utils/keyfile.c b/src/utils/keyfile.c index 9a9d8c8f..e1d8658f 100644 --- a/src/utils/keyfile.c +++ b/src/utils/keyfile.c @@ -142,3 +142,12 @@ void clean_keyfile(uint64_t unique_id) unlink(path); debug_printf("Cleaned keyfile %s\n", path); } + +const char* get_sec_keydir_path(void) +{ +#if defined(KEYFILE) && defined(SEC_KEYDIR) + return SEC_KEYDIR; +#else + return NULL; +#endif +} diff --git a/src/utils/keyfile.h b/src/utils/keyfile.h index 787eabd9..bfb92d06 100644 --- a/src/utils/keyfile.h +++ b/src/utils/keyfile.h @@ -27,6 +27,7 @@ void create_key(unsigned char *buffer, int key_size_bytes); void get_keyfile_path(char *pathname, int pathname_len, uint64_t unique_id); void create_keyfile(uint64_t unique_id); void clean_keyfile(uint64_t unique_id); +const char* get_sec_keydir_path(void); #if defined(__cplusplus) } diff --git a/testsuite/config_helper/README.md b/testsuite/config_helper/README.md index 5df01719..77e428d9 100644 --- a/testsuite/config_helper/README.md +++ b/testsuite/config_helper/README.md @@ -28,6 +28,10 @@ cd testsuite/config_helper 11. **test_11_pythonprefix_valid** - Validates Python installation (skips if /usr/tce missing) 12. **test_12_pythonprefix_invalid** - Warns when path lacks bin/python 13. **test_13_pythonprefix_undefined_var** - Detects undefined variables in pythonprefix +14. **test_14_security_mode_detection** - Detects and reports security configuration +15. **test_15_security_none_warning** - Warns when security=none (insecure) +16. **test_16_munge_detection** - Reports munge availability +17. **test_17_launchmon_warning** - Warns when security=launchmon (deprecated) ## Sample Mount Files diff --git a/testsuite/config_helper/run_tests.sh b/testsuite/config_helper/run_tests.sh index 75befe75..29358f02 100755 --- a/testsuite/config_helper/run_tests.sh +++ b/testsuite/config_helper/run_tests.sh @@ -124,6 +124,32 @@ test_13_pythonprefix_undefined_var() { echo "$output" | grep -q "Variable is not set" } +# Test 14: Security mode detection +test_14_security_mode_detection() { + output=$($HELPER --commpaths=/tmp 2>&1) + echo "$output" | grep -q "# Security configuration:" && \ + echo "$output" | grep -q "Current mode:" +} + +# Test 15: Security none warning +test_15_security_none_warning() { + output=$($HELPER --commpaths=/tmp --security=none 2>&1) + echo "$output" | grep -q "WARNING: No authentication enabled" +} + +# Test 16: Munge detection +test_16_munge_detection() { + output=$($HELPER --commpaths=/tmp 2>&1) + # Check that munge availability is reported + echo "$output" | grep -q "Munge availability:" +} + +# Test 17: LaunchMON warning +test_17_launchmon_warning() { + output=$($HELPER --commpaths=/tmp --security=launchmon 2>&1) + echo "$output" | grep -q "WARNING: LaunchMON security mode" +} + # Run all tests echo "=========================================" echo "spindle-config-helper Test Suite" @@ -144,6 +170,10 @@ run_test "10_relative_path_resolution" test_10_relative_path run_test "11_pythonprefix_valid" test_11_pythonprefix_valid run_test "12_pythonprefix_invalid" test_12_pythonprefix_invalid run_test "13_pythonprefix_undefined_var" test_13_pythonprefix_undefined_var +run_test "14_security_mode_detection" test_14_security_mode_detection +run_test "15_security_none_warning" test_15_security_none_warning +run_test "16_munge_detection" test_16_munge_detection +run_test "17_launchmon_warning" test_17_launchmon_warning echo "" echo "=========================================" From 286f3972717dad9f8b7ba6c914443bccc68221d4 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 30 Oct 2026 09:04:41 -0700 Subject: [PATCH 07/18] [config-helper] Rework per discussions. --- src/config_helper/man/spindle-config-helper.1 | 302 ++++- src/config_helper/spindle-config-helper.cc | 1131 ++++++++++++++++- 2 files changed, 1312 insertions(+), 121 deletions(-) diff --git a/src/config_helper/man/spindle-config-helper.1 b/src/config_helper/man/spindle-config-helper.1 index f275d3e0..97afb0c9 100644 --- a/src/config_helper/man/spindle-config-helper.1 +++ b/src/config_helper/man/spindle-config-helper.1 @@ -3,113 +3,295 @@ spindle-config-helper \- generate and validate Spindle configuration files .SH SYNOPSIS .B spindle-config-helper -[\fIOPTIONS\fR] +[\fIOPTIONS\fR] [\fB\-\-\fR \fISPINDLE_OPTIONS\fR] .SH DESCRIPTION .B spindle-config-helper -is a utility for generating and validating Spindle configuration files. -It parses configuration from command-line options, environment variables, and -existing configuration files, then outputs the resulting configuration in +is a utility for generating and validating Spindle configuration files in HPC +environments. It analyzes the local system to automatically detect optimal +configuration settings and outputs a .I spindle.conf -format. +file suitable for the environment. .PP -In future releases, this tool will also probe the runtime environment to -automatically detect optimal configuration settings, including: +The tool performs the following analysis: .IP \(bu 2 -Local mount points suitable for cache and communication paths +Detects local vs. remote filesystem mounts (NFS, Lustre, GPFS, tmpfs) .IP \(bu 2 -Available resource managers (Slurm, Flux, LSF) +Validates +.B $TMPDIR +availability and recommends appropriate paths .IP \(bu 2 -Security subsystems (Munge availability) +Tests SSH connectivity to verify compute node access .IP \(bu 2 -MPI environment characteristics +Discovers Python installations via system PATH +.IP \(bu 2 +Generates +.I localprefix +from top-level directories without remote mounts +.IP \(bu 2 +Validates path writability and accessibility .PP -The tool is designed to be run on compute nodes in the same environment where -Spindle will execute, as login/front-end nodes may have different characteristics -than compute nodes. +The tool should be run on compute nodes in the same environment where +Spindle will execute, as login/front-end nodes may have different characteristics. .SH OPTIONS .TP .BR \-h ", " \-\-help -Display usage information and exit. +Display detailed usage information and exit. .TP .BR \-V ", " \-\-version Display version information and exit. .TP +.BR \-o ", " \-\-output =\fIFILE\fR +Write output to +.I FILE +instead of stdout. By default, fails if +.I FILE +already exists. +.TP +.BR \-\-overwrite +Allow overwriting an existing output file when used with +.BR \-o . +.TP +.BR \-c ", " \-\-check [=\fIFILE\fR] +Validate an existing +.I spindle.conf +file. Checks option names against the current Spindle version, validates +enum values, and reports unknown or deprecated options. If +.I FILE +is not specified, only validates options passed after +.BR \-\- . +.TP +.BR \-a ", " \-\-all +Generate a complete configuration file including all Spindle options. +Options with default values are commented out (prefixed with '#'). +Options specified after +.B \-\- +are uncommented. Useful for discovering available options. +.TP .BR \-\-usage -Display a brief usage summary with all available options and exit. +Display Spindle's usage summary with all available configuration options. +.SH MODES +.SS Default Mode +Without any mode flags, +.B spindle-config-helper +generates a minimal configuration with four essential path variables: +.TP +.B commpaths +Local directories for inter-process communication. Prefers +.B $TMPDIR +if set, otherwise recommends a suitable local filesystem like +.I /tmp +or +.IR /dev/shm . +.TP +.B cachepaths +Local directories for file caching. Uses similar logic to +.BR commpaths . +.TP +.B pythonprefix +Python installation directories discovered via +.B which python +and +.BR "which python3" . +Validates that +.I bin/python +or +.I bin/python3 +exists under each directory. +.TP +.B localprefix +Colon-separated list of top-level directories that should be excluded from +caching. Generated from +.I /proc +.I /sys +.I /dev +and other top-level directories without remote mounts underneath. .PP -All Spindle configuration options are accepted. Use -.B --usage -to see a complete list of available options, or see the Spindle documentation -for detailed descriptions of each option. +The output includes warnings and recommendations as comments. +.SS Check Mode +When invoked with +.BR \-c , +the tool validates an existing configuration file: +.IP \(bu 2 +Parses +.I key = value +pairs from the file +.IP \(bu 2 +Verifies option names are valid in the current Spindle version +.IP \(bu 2 +Validates enum values (e.g., level must be high/medium/low/off) +.IP \(bu 2 +Reports malformed lines, unknown options, and invalid values +.IP \(bu 2 +Command-line options passed after +.B \-\- +take precedence over config file values +.SS All Mode +When invoked with +.BR \-a , +the tool enumerates all Spindle configuration options (~49 options). +Options with default values are commented out, while options specified +on the command line after +.B \-\- +are uncommented with their specified values. +.SH ENVIRONMENT +.TP +.B TMPDIR +Preferred directory for +.B commpaths +and +.BR cachepaths . +If not set, the tool recommends +.I /tmp +or another suitable local filesystem. +.TP +.B SANDBOX_ENGINE +If set, indicates the tool is running in a sandbox environment. +Affects capability testing (e.g., Munge daemon connectivity cannot be +verified in a sandbox). +.SH SPINDLE OPTIONS +Any valid Spindle configuration option can be passed after +.BR \-\- . +Use +.B spindle-config-helper \-\- \-\-usage +to see all available options. Common options include: +.TP +.BI \-\-commpaths= PATHS +Colon-separated list of directories for inter-process communication. +Should be on local (not remote) filesystems. +.TP +.BI \-\-cachepaths= PATHS +Colon-separated list of directories for file caching. +Should be on local filesystems with sufficient space. +.TP +.BI \-\-level= LEVEL +Aggressiveness of library redistribution: +.BR high ", " medium ", " low ", " off ", or " default . +.TP +.BI \-\-security= MODE +Security mode: +.BR none ", " munge ", " keyfile ", or " launchmon . +.TP +.BI \-\-launcher= TYPE +Resource manager: +.BR serial ", " slurm ", " flux ", " lsf ", " alps ", " openmpi ", " wreckrun ", " jsrun ", " hostbin ", or " srun . +.TP +.BI \-\-python\-prefix= PATHS +Colon-separated list of Python installation prefixes. +.TP +.BI \-\-local\-prefix= PATHS +Colon-separated list of directory prefixes to exclude from caching. .SH EXAMPLES .TP -Parse command-line options and output configuration: +Generate a new configuration to stdout: .nf .RS -spindle-config-helper --commpaths=/dev/shm --cachepaths=/tmp/cache +spindle-config-helper .RE .fi .TP -Parse an existing configuration file: +Generate and save to file: .nf .RS -spindle-config-helper --config=/etc/spindle.conf +spindle-config-helper -o ~/.spindle/spindle.conf .RE .fi .TP -Combine multiple configuration sources: +Generate with custom paths: .nf .RS -export SPINDLE_SECURITY=none -spindle-config-helper --config=/etc/spindle.conf --launcher=slurm +spindle-config-helper -- --commpaths=/dev/shm --cachepaths=/tmp +.RE +.fi +.TP +Check existing config for warnings: +.nf +.RS +spindle-config-helper -c ~/.spindle/spindle.conf | grep WARNING +.RE +.fi +.TP +Update existing config with new security setting: +.nf +.RS +spindle-config-helper -c ~/.spindle/spindle.conf \\ + -o ~/.spindle/spindle.conf --overwrite \\ + -- --security=munge +.RE +.fi +.TP +Generate complete config with all options: +.nf +.RS +spindle-config-helper -a -o spindle.conf.full +.RE +.fi +.TP +Generate config with high aggressiveness: +.nf +.RS +spindle-config-helper -a -- --level=high --launcher=slurm +.RE +.fi +.TP +View all available Spindle options: +.nf +.RS +spindle-config-helper -- --usage .RE .fi -.PP -The tool merges configuration from multiple sources in the following order -(later sources override earlier ones): -.IP 1. 3 -Built-in defaults -.IP 2. 3 -System configuration file -.IP 3. 3 -User configuration file -.IP 4. 3 -Environment variables -.IP 5. 3 -Command-line options -.SH FUTURE FUNCTIONALITY -Future versions will support: -.TP -.B Environment Probing -Automatically detect and recommend optimal settings for cache paths, communication -paths, and other configuration based on the compute node environment. -.TP -.B Configuration Validation -Verify that specified paths exist, are writable, and are on appropriate filesystem -types (e.g., local vs. network filesystems). -.TP -.B Performance Tuning -Analyze the environment and suggest performance-oriented configuration settings. -.TP -.B MPI Integration -Run as an MPI application to gather environment information across all compute nodes -in an allocation. .SH FILES .TP .I /etc/spindle.conf System-wide Spindle configuration file. .TP .I ~/.spindle/spindle.conf -Per-user Spindle configuration file. +Per-user Spindle configuration file (common location). +.TP +.I /proc/mounts +Used to detect filesystem mount points and types. .SH EXIT STATUS .TP .B 0 -Success. Valid configuration was parsed and output. +Success. Valid configuration was generated or validated. .TP .B 1 -Error. Invalid configuration or parse error occurred. +Error. Invalid arguments, file I/O error, or configuration parse error occurred. +.SH NOTES +.SS Path Validation +The tool validates paths by checking: +.IP \(bu 2 +Parent directory is writable +.IP \(bu 2 +Path is on a local (not remote) filesystem +.IP \(bu 2 +Path can be created and deleted (test file creation) +.PP +Paths that fail validation are filtered out and reported in warnings. +.SS SSH Testing +The tool tests SSH connectivity to the local hostname to verify that +compute node access is properly configured. This test uses +.BR "ssh -o BatchMode=yes" +to check for key-based authentication without password prompts. +.SS Python Discovery +Python installations are discovered by running +.B which python +and +.BR "which python3" , +following symlinks to find the actual installation directories. +The tool validates that +.I bin/python +or +.I bin/python3 +exists under each discovered prefix. +.SS Sandbox Environments +When running in a sandbox environment (detected via +.BR SANDBOX_ENGINE ), +some tests are limited. For example, Munge daemon connectivity cannot +be verified, though Munge binary availability is still checked. .SH SEE ALSO .BR spindle (1) +.PP +Spindle documentation: https://github.com/hpc/Spindle .SH AUTHOR The Spindle development team at Lawrence Livermore National Laboratory. .SH BUGS diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 71564559..52bae440 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -31,6 +31,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include +#include +#include extern "C" { #include "keyfile.h" @@ -340,19 +343,21 @@ static PathAnalysis analyze_path(const std::string& path, const std::vector& moun analyze_option(confLocalPrefix, "localprefix"); } +static std::string get_timestamp() +{ + time_t now = time(nullptr); + char buf[80]; + strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", localtime(&now)); + return std::string(buf); +} + +static bool test_directory_writable(const std::string& path) +{ + // Try to create a test file in the directory + std::string testfile = path + "/.spindle_config_test_" + std::to_string(getpid()); + + int fd = open(testfile.c_str(), O_CREAT | O_WRONLY | O_EXCL, 0600); + if (fd < 0) { + return false; + } + + close(fd); + unlink(testfile.c_str()); + return true; +} + +static bool validate_tmpdir_path(const std::string& path, + const std::vector& mounts, + std::string& error_msg) +{ + // Resolve the path (in case it's a symlink) + std::string resolved = resolve_real_path(path); + + // Check if path exists + struct stat st; + if (stat(resolved.c_str(), &st) != 0) { + error_msg = "Path does not exist: " + path; + if (resolved != path) { + error_msg += " (resolves to " + resolved + ")"; + } + return false; + } + + // Check if it's a directory + if (!S_ISDIR(st.st_mode)) { + error_msg = "Not a directory: " + path; + return false; + } + + // Check if writable + if (!test_directory_writable(resolved)) { + error_msg = "Directory is not writable: " + path; + return false; + } + + // Check if on local filesystem + const MountPoint* mp = find_mount_for_path(resolved, mounts); + if (!mp) { + error_msg = "Cannot determine filesystem type for: " + path; + return false; + } + + if (mp->is_remote) { + error_msg = "Path is on REMOTE filesystem (" + mp->fstype + "): " + path; + return false; + } + + return true; +} + +static std::string determine_best_path(const std::vector& mounts, + std::vector& warnings) +{ + // Check $TMPDIR first + const char* tmpdir_env = getenv("TMPDIR"); + + if (tmpdir_env && tmpdir_env[0] != '\0') { + std::string tmpdir = tmpdir_env; + std::string error_msg; + + if (validate_tmpdir_path(tmpdir, mounts, error_msg)) { + return tmpdir; + } else { + warnings.push_back("$TMPDIR is set to '" + tmpdir + "' but not usable: " + error_msg); + } + } else { + warnings.push_back("$TMPDIR is not set"); + } + + // Fall back to /tmp + std::string error_msg; + if (validate_tmpdir_path("/tmp", mounts, error_msg)) { + return "/tmp"; + } else { + warnings.push_back("/tmp is not usable: " + error_msg); + } + + // No suitable path found + return ""; +} + +static std::string get_python_prefix_from_which(const char* python_cmd) +{ + // Run 'which python' or 'which python3' + std::string cmd = std::string("which ") + python_cmd + " 2>/dev/null"; + FILE* fp = popen(cmd.c_str(), "r"); + if (!fp) { + return ""; + } + + char path[PATH_MAX]; + std::string result; + if (fgets(path, sizeof(path), fp) != nullptr) { + // Remove trailing newline + size_t len = strlen(path); + if (len > 0 && path[len-1] == '\n') { + path[len-1] = '\0'; + } + + // Resolve symlinks + char resolved[PATH_MAX]; + if (realpath(path, resolved) != nullptr) { + // Get parent directory (strip /bin/python*) + std::string full_path = resolved; + size_t bin_pos = full_path.rfind("/bin/"); + if (bin_pos != std::string::npos) { + result = full_path.substr(0, bin_pos); + } + } + } + pclose(fp); + return result; +} + +static std::vector discover_python_prefixes() +{ + std::set prefixes; + + // Try 'which python' + std::string py2 = get_python_prefix_from_which("python"); + if (!py2.empty()) { + prefixes.insert(py2); + } + + // Try 'which python3' + std::string py3 = get_python_prefix_from_which("python3"); + if (!py3.empty()) { + prefixes.insert(py3); + } + + return std::vector(prefixes.begin(), prefixes.end()); +} + +static bool has_remote_mount_under(const std::string& prefix, const std::vector& mounts) +{ + // Check if any remote mount exists under this prefix + std::string prefix_slash = prefix; + if (prefix != "/" && prefix_slash.back() != '/') { + prefix_slash += "/"; + } + + for (const auto& mp : mounts) { + if (!mp.is_remote) continue; + + // Check if this remote mount is under prefix + if (mp.path == prefix || mp.path.find(prefix_slash) == 0) { + return true; + } + } + return false; +} + +static std::vector generate_localprefix(const std::vector& mounts) +{ + std::vector local_dirs; + + // Always include kernel filesystems + local_dirs.push_back("/proc"); + local_dirs.push_back("/sys"); + local_dirs.push_back("/dev"); + + // Get top-level directories + DIR* dir = opendir("/"); + if (!dir) { + return local_dirs; + } + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_name[0] == '.') continue; + + std::string top_level = std::string("/") + entry->d_name; + + // Skip if already added (kernel filesystems) + if (top_level == "/proc" || top_level == "/sys" || top_level == "/dev") { + continue; + } + + // Check if directory + struct stat st; + if (stat(top_level.c_str(), &st) != 0 || !S_ISDIR(st.st_mode)) { + continue; + } + + // Skip if has remote mounts underneath + if (has_remote_mount_under(top_level, mounts)) { + continue; + } + + local_dirs.push_back(top_level); + } + closedir(dir); + + // Sort for consistent output (kernel filesystems first) + std::sort(local_dirs.begin() + 3, local_dirs.end()); + + return local_dirs; +} + +static std::string get_hostname_for_ssh() +{ + char hostname[256]; + if (gethostname(hostname, sizeof(hostname)) != 0) { + return ""; + } + return std::string(hostname); +} + +static bool test_ssh_connectivity(const std::string& hostname, std::string& error_msg) +{ + if (hostname.empty()) { + error_msg = "Could not determine hostname"; + return false; + } + + // Build SSH command: ssh -o BatchMode=yes -o ConnectTimeout=10 /bin/true + std::string cmd = "ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no " + + hostname + " /bin/true 2>&1"; + + FILE* fp = popen(cmd.c_str(), "r"); + if (!fp) { + error_msg = "Failed to execute ssh command"; + return false; + } + + // Read any error output + char buffer[256]; + std::string output; + while (fgets(buffer, sizeof(buffer), fp) != nullptr) { + output += buffer; + } + + int status = pclose(fp); + int exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; + + if (exit_code == 0) { + return true; + } + + // SSH failed - extract useful error info + if (output.find("Permission denied") != std::string::npos) { + error_msg = "SSH authentication failed (check SSH keys)"; + } else if (output.find("Connection refused") != std::string::npos) { + error_msg = "SSH connection refused"; + } else if (output.find("Connection timed out") != std::string::npos) { + error_msg = "SSH connection timed out"; + } else if (output.find("No route to host") != std::string::npos) { + error_msg = "No route to host"; + } else if (!output.empty()) { + // Truncate long error messages + error_msg = output.substr(0, 100); + if (output.length() > 100) error_msg += "..."; + } else { + error_msg = "SSH test failed (exit code " + std::to_string(exit_code) + ")"; + } + + return false; +} + +// Parse a spindle.conf file and return key-value pairs +// Returns: map of key -> value pairs +// Ignores comments and blank lines +static std::map parse_config_file(const std::string& filepath, + std::vector& warnings) +{ + std::map config; + std::ifstream file(filepath); + + if (!file.is_open()) { + warnings.push_back("WARNING: Could not open config file: " + filepath); + return config; + } + + std::string line; + int line_num = 0; + while (std::getline(file, line)) { + line_num++; + + // Trim leading/trailing whitespace + size_t start = line.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) continue; // Empty line + + size_t end = line.find_last_not_of(" \t\r\n"); + line = line.substr(start, end - start + 1); + + // Skip comments and empty lines + if (line.empty() || line[0] == '#') continue; + + // Find '=' separator + size_t eq_pos = line.find('='); + if (eq_pos == std::string::npos) { + warnings.push_back("WARNING: Malformed line " + std::to_string(line_num) + + " in " + filepath + ": " + line); + continue; + } + + // Extract key and value + std::string key = line.substr(0, eq_pos); + std::string value = line.substr(eq_pos + 1); + + // Trim whitespace around key and value + start = key.find_first_not_of(" \t"); + end = key.find_last_not_of(" \t"); + if (start != std::string::npos) { + key = key.substr(start, end - start + 1); + } + + start = value.find_first_not_of(" \t"); + end = value.find_last_not_of(" \t"); + if (start != std::string::npos) { + value = value.substr(start, end - start + 1); + } + + if (!key.empty()) { + config[key] = value; + } + } + + file.close(); + return config; +} + +static void print_config_header(std::vector& warnings) +{ + std::cout << "# spindle.conf generated by spindle-config-helper version " + << PACKAGE_VERSION << std::endl; + std::cout << "# Generated: " << get_timestamp() << std::endl; + std::cout << "#" << std::endl; + + if (!warnings.empty()) { + // Check if we have any actual warnings vs just informational notes + bool has_warnings = false; + for (const auto& msg : warnings) { + if (msg.find("WARNING") != std::string::npos || + msg.find("CRITICAL") != std::string::npos || + msg.find("failed") != std::string::npos || + msg.find("not set") != std::string::npos || + msg.find("not usable") != std::string::npos) { + has_warnings = true; + break; + } + } + + std::cout << "# " << (has_warnings ? "WARNINGS:" : "NOTES:") << std::endl; + for (const auto& warning : warnings) { + std::cout << "# " << warning << std::endl; + } + std::cout << "#" << std::endl; + } +} + +static void print_path_variable(const std::string& name, const std::string& value, + const std::vector& comments) +{ + // Print comments above the variable + for (const auto& comment : comments) { + std::cout << "# " << comment << std::endl; + } + + std::cout << name << " = " << value << std::endl; + + if (!comments.empty()) { + std::cout << std::endl; // Blank line after commented variables + } +} + +static void generate_default_config(const ConfigMap& confmap, + const std::vector& mounts, + std::vector& header_warnings) +{ + // Determine best path for commpaths/cachepaths if using defaults + std::string recommended_path; + bool need_path_recommendation = false; + + // Check if commpaths or cachepaths are using the default $TMPDIR + auto commpaths_result = confmap.getValueString(confCommPaths); + auto cachepaths_result = confmap.getValueString(confCachePaths); + + bool commpaths_is_default = (commpaths_result.first && commpaths_result.second == "$TMPDIR"); + bool cachepaths_is_default = (cachepaths_result.first && cachepaths_result.second == "$TMPDIR"); + + if (commpaths_is_default || cachepaths_is_default) { + need_path_recommendation = true; + std::vector path_warnings; + recommended_path = determine_best_path(mounts, path_warnings); + + // Add path-related warnings to header + for (const auto& warning : path_warnings) { + header_warnings.push_back(warning); + } + + if (recommended_path.empty()) { + header_warnings.push_back("CRITICAL: No suitable local filesystem found for commpaths/cachepaths"); + header_warnings.push_back(" Cannot generate valid configuration"); + // Print header with errors and exit + print_config_header(header_warnings); + return; + } + } + + // Test SSH connectivity + std::string hostname = get_hostname_for_ssh(); + std::string ssh_error; + if (!test_ssh_connectivity(hostname, ssh_error)) { + header_warnings.push_back("SSH test failed for " + hostname + ": " + ssh_error); + header_warnings.push_back(" (SSH keys may not be configured for compute node access)"); + } else { + header_warnings.push_back("SSH test passed for " + hostname); + } + + // Header + print_config_header(header_warnings); + + // Generate the 4 path-based variables with analysis + auto output_path_option = [&](SpindleConfigID config_id, const char* name, + bool check_python = false) { + // Check if this option is set (either default or user-specified) + if (!confmap.isSet(config_id)) { + return; + } + + auto result = confmap.getValueString(config_id); + if (!result.first || result.second.empty()) { + return; + } + + std::string paths = result.second; + std::vector comments; + + // If this is commpaths or cachepaths with $TMPDIR default, use recommended path + if (need_path_recommendation && + (config_id == confCommPaths || config_id == confCachePaths) && + paths == "$TMPDIR" && !recommended_path.empty()) { + paths = recommended_path; + } + + // For pythonprefix, discover additional Python installations + std::vector discovered_pythons; + if (config_id == confPythonPrefix) { + discovered_pythons = discover_python_prefixes(); + } + + // Analyze each path in the colon-separated list + std::vector path_list; + std::vector analyses; + size_t pos = 0; + while (pos < paths.length()) { + size_t end = paths.find(':', pos); + if (end == std::string::npos) { + end = paths.length(); + } + std::string path = paths.substr(pos, end - pos); + if (!path.empty()) { + path_list.push_back(path); + analyses.push_back(analyze_path(path, mounts, check_python)); + } + pos = end + 1; + } + + // Add discovered Python paths to the list + if (config_id == confPythonPrefix) { + for (const auto& pypath : discovered_pythons) { + // Check if not already in list + bool already_present = false; + for (const auto& existing : path_list) { + PathAnalysis ex = analyze_path(existing, mounts, check_python); + if (ex.resolved == resolve_real_path(pypath)) { + already_present = true; + break; + } + } + if (!already_present) { + path_list.push_back(pypath); + analyses.push_back(analyze_path(pypath, mounts, check_python)); + comments.push_back("NOTE: Added " + pypath + " from 'which python/python3'"); + } + } + } + + // Separate usable from unusable paths + std::vector good_paths, bad_paths; + + for (size_t i = 0; i < path_list.size(); i++) { + bool has_critical_issues = false; + + // For pythonprefix, check for Python installation issues + if (check_python) { + for (const auto& warning : analyses[i].warnings) { + if (warning.find("No bin/python") != std::string::npos || + warning.find("Path does not exist") != std::string::npos || + warning.find("parent directory is not writable") != std::string::npos) { + has_critical_issues = true; + break; + } + } + } else { + // For commpaths/cachepaths/localprefix + for (const auto& warning : analyses[i].warnings) { + if (warning.find("parent directory is not writable") != std::string::npos || + warning.find("REMOTE filesystem") != std::string::npos) { + has_critical_issues = true; + break; + } + } + } + + if (has_critical_issues) { + bad_paths.push_back(path_list[i]); + } else { + good_paths.push_back(path_list[i]); + } + } + + // If no good paths, add fallback + if (good_paths.empty() && (config_id == confCommPaths || config_id == confCachePaths)) { + if (!recommended_path.empty()) { + good_paths.push_back(recommended_path); + comments.push_back("NOTE: No usable paths found, using " + recommended_path); + } + } + + // Build comments for bad paths (these came from configure/defaults, not user choice) + if (!bad_paths.empty()) { + if (check_python) { + comments.push_back("NOTE: The following paths are not valid Python installations:"); + } else { + comments.push_back("NOTE: The following paths from configure/defaults are not usable:"); + } + for (size_t i = 0; i < path_list.size(); i++) { + bool is_bad = (std::find(bad_paths.begin(), bad_paths.end(), path_list[i]) != bad_paths.end()); + if (is_bad) { + comments.push_back(" " + path_list[i] + ":"); + for (const auto& warning : analyses[i].warnings) { + comments.push_back(" " + warning); + } + } + } + if (!good_paths.empty()) { + comments.push_back(" (These paths have been filtered out)"); + } + } + + // Use only good paths in output + if (!good_paths.empty()) { + std::string filtered; + for (const auto& p : good_paths) { + if (!filtered.empty()) filtered += ":"; + filtered += p; + } + paths = filtered; + } else { + // No good paths at all + if (check_python) { + // For pythonprefix, skip output if no valid Python found + comments.push_back("WARNING: No valid Python installations found"); + return; + } + paths = ""; + } + + // Add comment for usable paths if no issues + bool has_issues = !bad_paths.empty(); + if (!has_issues && !good_paths.empty()) { + PathAnalysis first = analyze_path(good_paths[0], mounts, check_python); + if (first.mount && !first.mount->is_remote) { + comments.push_back(std::string(name) + ": Local " + first.mount->fstype + + " filesystem, " + std::to_string(first.mount->avail_gb) + "GB available"); + } else if (is_kernel_filesystem(first.resolved)) { + comments.push_back(std::string(name) + ": Kernel filesystem"); + } + } + + print_path_variable(name, paths, comments); + }; + + output_path_option(confCommPaths, "commpaths"); + output_path_option(confCachePaths, "cachepaths"); + output_path_option(confPythonPrefix, "pythonprefix", true); + + // Special handling for localprefix - generate from top-level dirs without remote mounts + auto localprefix_result = confmap.getValueString(confLocalPrefix); + if (localprefix_result.first) { + std::vector generated_locals = generate_localprefix(mounts); + std::vector comments; + + // Check if user specified additional paths (beyond defaults) + std::string user_value = localprefix_result.second; + std::set user_paths; + + // Parse user's colon-separated list + size_t pos = 0; + while (pos < user_value.length()) { + size_t end = user_value.find(':', pos); + if (end == std::string::npos) end = user_value.length(); + + std::string path = user_value.substr(pos, end - pos); + // Strip trailing slash for comparison + if (!path.empty() && path.back() == '/') path.pop_back(); + if (!path.empty()) user_paths.insert(path); + + pos = end + 1; + } + + // Add generated paths to user paths + std::set all_paths(user_paths); + for (const auto& gpath : generated_locals) { + all_paths.insert(gpath); + } + + // Validate user additions + for (const auto& upath : user_paths) { + // Check if this was NOT auto-generated (meaning user added it) + bool was_generated = (std::find(generated_locals.begin(), generated_locals.end(), upath) != generated_locals.end()); + if (!was_generated) { + // User added this - validate it + if (has_remote_mount_under(upath, mounts)) { + comments.push_back("WARNING: User-specified path has remote mounts underneath: " + upath); + } else { + comments.push_back("NOTE: User added: " + upath); + } + } + } + + // Build output with trailing slashes + std::string output_value; + for (const auto& dir : all_paths) { + if (!output_value.empty()) output_value += ":"; + output_value += (dir == "/" ? "/" : dir + "/"); + } + + if (comments.empty()) { + comments.push_back("localprefix: Top-level directories without remote mounts"); + } + + print_path_variable("localprefix", output_value, comments); + } +} + +static void generate_all_config(const ConfigMap& confmap, + const std::vector& mounts, + std::vector& header_warnings) +{ + // Print header + print_config_header(header_warnings); + + // Get all Spindle options + const std::map& all_opts = getOptionsByID(); + + // Iterate through all options and output them + for (const auto& pair : all_opts) { + SpindleConfigID id = pair.first; + const SpindleOption& opt = pair.second; + + // Skip options without command-line names (internal options) + if (!opt.cmdline_long) continue; + + // Get the config file key name (cmdline_long with dashes removed) + std::string key = opt.cmdline_long; + key.erase(std::remove(key.begin(), key.end(), '-'), key.end()); + + // Check if this option is set (either by user or default) + bool is_set = confmap.isSet(id); + std::string value; + + if (is_set) { + // Get the actual value + auto result = confmap.getValueString(id); + if (result.first) { + value = result.second; + } + } + + // For the 4 path variables, output with full analysis + if (id == confCommPaths || id == confCachePaths || + id == confPythonPrefix || id == confLocalPrefix) { + // Use the existing detailed output from generate_default_config + continue; // We'll call those separately + } + + // For other options, output as comment if not explicitly set by user + // or uncommented if set by command line + std::string comment_prefix = is_set ? "" : "# "; + + // Output the option + std::cout << comment_prefix << key << " = " << value << std::endl; + } + + // Now output the 4 path variables with full analysis + // These are always uncommented and analyzed + auto output_path_option = [&](SpindleConfigID config_id, const char* name, + bool check_python = false) { + if (!confmap.isSet(config_id)) { + return; + } + + auto result = confmap.getValueString(config_id); + if (!result.first || result.second.empty()) { + return; + } + + std::string paths = result.second; + std::vector comments; + + // For pythonprefix, discover additional Python installations + std::vector discovered_pythons; + if (config_id == confPythonPrefix) { + discovered_pythons = discover_python_prefixes(); + } + + // Analyze paths (simplified for all mode) + std::vector path_list; + size_t pos = 0; + while (pos < paths.length()) { + size_t end = paths.find(':', pos); + if (end == std::string::npos) { + end = paths.length(); + } + std::string path = paths.substr(pos, end - pos); + if (!path.empty()) { + path_list.push_back(path); + } + pos = end + 1; + } + + // Add discovered Python paths + if (config_id == confPythonPrefix) { + for (const auto& pypath : discovered_pythons) { + if (std::find(path_list.begin(), path_list.end(), pypath) == path_list.end()) { + path_list.push_back(pypath); + } + } + } + + // Output + std::string output_value; + for (size_t i = 0; i < path_list.size(); i++) { + if (i > 0) output_value += ":"; + output_value += path_list[i]; + } + + if (!output_value.empty()) { + print_path_variable(name, output_value, comments); + } + }; + + output_path_option(confCommPaths, "commpaths"); + output_path_option(confCachePaths, "cachepaths"); + output_path_option(confPythonPrefix, "pythonprefix", true); + output_path_option(confLocalPrefix, "localprefix"); +} + static void print_version() { std::cout << "spindle-config-helper version " << PACKAGE_VERSION << std::endl; @@ -609,19 +1382,81 @@ static void print_security_analysis(const ConfigMap& confmap, const std::vector< static void print_usage(const char *progname) { - std::cout << "Usage: " << progname << " [OPTIONS]\n\n" - << "Parse Spindle configuration from command line, environment variables, and\n" - << "configuration files, then output the resulting configuration in spindle.conf format.\n\n" - << "Options:\n" + std::cout << "Usage: " << progname << " [OPTIONS] [-- SPINDLE_OPTIONS]\n\n" + << "Generate and validate Spindle configuration files for HPC environments.\n\n" + << "DESCRIPTION:\n" + << " spindle-config-helper analyzes the local system and generates appropriate\n" + << " spindle.conf files. It detects local vs. remote filesystems, discovers\n" + << " Python installations, validates paths, and tests SSH connectivity.\n\n" + << "OPTIONS:\n" << " -h, --help Display this help message and exit\n" << " -V, --version Display version information and exit\n" - << " --usage Display brief usage summary and exit\n\n" - << "All Spindle configuration options are supported. See spindle-config-helper(1)\n" - << "for details, or use --usage for a complete list of options.\n\n" - << "Examples:\n" - << " " << progname << " --commpaths=/dev/shm --cachepaths=/tmp\n" - << " " << progname << " --config=/etc/spindle.conf\n" - << " " << progname << " --launcher=slurm --security=none\n" + << " -o, --output=FILE Write output to FILE instead of stdout\n" + << " --overwrite Allow overwriting existing output file\n" + << " -c, --check[=FILE] Validate existing config FILE (optional)\n" + << " -a, --all Generate complete config with all options\n" + << " --usage Display Spindle's usage summary\n\n" + << "MODES:\n" + << " Default mode (no options):\n" + << " - Analyzes filesystem mounts (local vs. remote)\n" + << " - Checks $TMPDIR availability and recommends paths\n" + << " - Tests SSH connectivity to verify compute node access\n" + << " - Discovers Python installations via 'which python/python3'\n" + << " - Generates localprefix from top-level directories\n" + << " - Outputs minimal spindle.conf with 4 path variables:\n" + << " * commpaths: Local directories for inter-process communication\n" + << " * cachepaths: Local directories for file caching\n" + << " * pythonprefix: Python installation directories\n" + << " * localprefix: Directories excluded from caching\n" + << " - Includes warnings and recommendations as comments\n\n" + << " Check mode (-c FILE):\n" + << " - Reads and parses existing spindle.conf file\n" + << " - Validates option names against current Spindle version\n" + << " - Validates enum values (e.g., level, security, launcher)\n" + << " - Reports unknown or deprecated options\n" + << " - Command-line options after -- override config file values\n\n" + << " All mode (-a):\n" + << " - Enumerates all Spindle configuration options\n" + << " - Comments out options with default values (prefix '#')\n" + << " - Uncomments options specified after --\n" + << " - Useful for discovering available options\n\n" + << "ENVIRONMENT:\n" + << " $TMPDIR Preferred directory for commpaths/cachepaths\n" + << " Falls back to /tmp if not set\n\n" + << "SPINDLE_OPTIONS:\n" + << " Any valid Spindle option can be passed after --\n" + << " Use '" << progname << " -- --usage' to see all available options\n" + << " Common options:\n" + << " --commpaths=PATHS Colon-separated list of comm directories\n" + << " --cachepaths=PATHS Colon-separated list of cache directories\n" + << " --level=LEVEL Aggressiveness: high, medium, low, off\n" + << " --security=MODE Security: none, munge, keyfile\n" + << " --launcher=TYPE Launcher: serial, slurm, flux, etc.\n\n" + << "EXAMPLES:\n" + << " # Generate new config to stdout\n" + << " " << progname << "\n\n" + << " # Generate config for review before saving\n" + << " " << progname << " | less\n\n" + << " # Save config to file\n" + << " " << progname << " -o ~/.spindle/spindle.conf\n\n" + << " # Generate with custom paths\n" + << " " << progname << " -- --commpaths=/dev/shm --cachepaths=/tmp\n\n" + << " # Check existing config for warnings\n" + << " " << progname << " -c ~/.spindle/spindle.conf | grep WARNING\n\n" + << " # Update config with new security setting\n" + << " " << progname << " -c ~/.spindle/spindle.conf -o ~/.spindle/spindle.conf \\\n" + << " --overwrite -- --security=munge\n\n" + << " # Generate complete config with all options\n" + << " " << progname << " -a -o spindle.conf.full\n\n" + << " # See all available Spindle options\n" + << " " << progname << " -- --usage\n\n" + << "EXIT STATUS:\n" + << " 0 Success\n" + << " 1 Error (invalid arguments, file I/O error, etc.)\n\n" + << "SEE ALSO:\n" + << " spindle(1), spindle-config-helper(1)\n\n" + << "VERSION:\n" + << " spindle-config-helper " << PACKAGE_VERSION << "\n" << std::endl; } @@ -681,42 +1516,145 @@ static void format_config_value(const ConfigMap &confmap, SpindleConfigID id, int main(int argc, char *argv[]) { // Handle -h/--help and -V/--version before config parsing + // Also check for -- --usage which should show Spindle's usage + bool show_spindle_usage = false; + int separator_idx = -1; + std::string output_file; + bool overwrite = false; + std::string check_file; + bool check_mode = false; + bool all_mode = false; + + // First pass: check for help/version and find separator, parse output options for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - print_usage(argv[0]); - return 0; + if (strcmp(argv[i], "--") == 0) { + separator_idx = i; + continue; } - if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) { - print_version(); - return 0; + + if (separator_idx == -1) { + // Before --: check for config-helper options + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + print_usage(argv[0]); + return 0; + } + if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) { + print_version(); + return 0; + } + if (strcmp(argv[i], "-o") == 0 && i + 1 < argc) { + output_file = argv[i + 1]; + i++; // Skip next argument + } + if (strncmp(argv[i], "--output=", 9) == 0) { + output_file = argv[i] + 9; + } + if (strcmp(argv[i], "--output") == 0 && i + 1 < argc) { + output_file = argv[i + 1]; + i++; // Skip next argument + } + if (strcmp(argv[i], "--overwrite") == 0) { + overwrite = true; + } + if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--all") == 0) { + all_mode = true; + } + if (strcmp(argv[i], "-c") == 0) { + check_mode = true; + // Check if next arg is a file (not starting with -) + if (i + 1 < argc && argv[i + 1][0] != '-') { + check_file = argv[i + 1]; + i++; // Skip next argument + } + } + if (strncmp(argv[i], "--check=", 8) == 0) { + check_mode = true; + check_file = argv[i] + 8; + } + if (strcmp(argv[i], "--check") == 0) { + check_mode = true; + if (i + 1 < argc && argv[i + 1][0] != '-') { + check_file = argv[i + 1]; + i++; // Skip next argument + } + } + } else { + // After --: check for --usage + if (strcmp(argv[i], "--usage") == 0) { + show_spindle_usage = true; + } } } - // Check if application command was provided (after '--') - bool has_app = false; - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--") == 0 && i + 1 < argc) { - has_app = true; - break; + // Reconstruct argv to move Spindle options before -- and add dummy app + // Input: spindle-config-helper [config-helper-opts] -- [spindle-opts] + // Output: spindle-config-helper [spindle-opts] -- /bin/true (filtering out config-helper opts) + + std::vector new_argv; + new_argv.push_back(argv[0]); // Program name + + auto is_config_helper_opt = [](const char* arg) { + return strcmp(arg, "-o") == 0 || + strcmp(arg, "-c") == 0 || + strcmp(arg, "-a") == 0 || + strncmp(arg, "--output", 8) == 0 || + strncmp(arg, "--check", 7) == 0 || + strcmp(arg, "--all") == 0 || + strcmp(arg, "--overwrite") == 0; + }; + + if (separator_idx != -1) { + // Add config-helper options (before --), filtering out our options + for (int i = 1; i < separator_idx; i++) { + if (is_config_helper_opt(argv[i])) { + // Skip this arg, and if it takes an argument, skip that too + if ((strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0 || + strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--check") == 0) && + i + 1 < argc && argv[i + 1][0] != '-') { + i++; // Skip the file argument too + } + continue; + } + new_argv.push_back(argv[i]); + } + + // Add spindle options (after --) + for (int i = separator_idx + 1; i < argc; i++) { + new_argv.push_back(argv[i]); + } + } else { + // No -- found, copy args but filter out config-helper options + for (int i = 1; i < argc; i++) { + if (is_config_helper_opt(argv[i])) { + if ((strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0 || + strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--check") == 0) && + i + 1 < argc && argv[i + 1][0] != '-') { + i++; // Skip the file argument too + } + continue; + } + new_argv.push_back(argv[i]); } } - // If no application provided, add a dummy one for the parser - char **modified_argv = argv; - int modified_argc = argc; - char *dummy_app = nullptr; + // Add dummy app + char *dummy_sep = strdup("--"); + char *dummy_app = strdup("/bin/true"); + new_argv.push_back(dummy_sep); + new_argv.push_back(dummy_app); - if (!has_app) { - dummy_app = strdup("/bin/true"); - modified_argv = new char*[argc + 2]; - for (int i = 0; i < argc; i++) { - modified_argv[i] = argv[i]; - } - modified_argv[argc] = strdup("--"); - modified_argv[argc + 1] = dummy_app; - modified_argc = argc + 2; + // Handle check mode: parse config file if provided + std::map check_config; + std::vector check_warnings; + + if (check_mode && !check_file.empty()) { + check_config = parse_config_file(check_file, check_warnings); + // Config values will be merged with confmap after parsing command-line options } + char **modified_argv = new_argv.data(); + int modified_argc = new_argv.size(); + ConfigMap confmap; std::string errmsg; @@ -724,35 +1662,106 @@ int main(int argc, char *argv[]) // Pass false for inExecutable since we're not the spindle launcher if (!gatherAllConfigInfo(modified_argc, modified_argv, false, confmap, errmsg)) { std::cerr << "Error: " << errmsg << std::endl; - if (!has_app) { - free(dummy_app); - free(modified_argv[argc]); - delete[] modified_argv; - } + free(dummy_app); + free(dummy_sep); return 1; } - // Clean up temporary argv if we created one - if (!has_app) { - free(dummy_app); - free(modified_argv[argc]); - delete[] modified_argv; + // Clean up temporary argv + free(dummy_app); + free(dummy_sep); + + // In check mode, merge config file values with confmap + // Command-line options take precedence (they're already in confmap) + if (check_mode && !check_config.empty()) { + // For each config file option, check if it's already set + // If not set, add it to confmap + for (const auto& kv : check_config) { + const std::string& key = kv.first; + const std::string& value = kv.second; + + // Find the corresponding SpindleConfigID + const std::map& all_opts = getOptionsByID(); + bool found = false; + SpindleConfigID found_id; + + for (const auto& pair : all_opts) { + const SpindleOption& opt = pair.second; + if (opt.cmdline_long) { + // Remove dashes from cmdline_long to match config file format + std::string opt_name = opt.cmdline_long; + opt_name.erase(std::remove(opt_name.begin(), opt_name.end(), '-'), opt_name.end()); + if (opt_name == key) { + found_id = opt.config_id; + found = true; + break; + } + } + } + + if (!found) { + check_warnings.push_back("WARNING: Unknown option in config file: " + key); + continue; + } + + // Validate the value from config file + // Note: Full merging with command-line overrides needs deeper integration + // For now, we just validate and report warnings + std::string errstring; + if (!confmap.set(found_id, value, errstring)) { + check_warnings.push_back("WARNING: Invalid value for " + key + " in config file: " + errstring); + } + } } // Parse and analyze mount points std::vector mounts = parse_mounts(); - // Print security analysis - print_security_analysis(confmap, mounts); + // Collect header warnings (will add $TMPDIR and SSH checks later) + std::vector header_warnings; + + // Add any warnings from config file parsing + header_warnings.insert(header_warnings.end(), check_warnings.begin(), check_warnings.end()); - // Print filesystem recommendations and warnings - print_filesystem_recommendations(mounts, confmap); + // Handle output file if specified + FILE* output_fp = nullptr; + int saved_stdout = -1; - // Output configuration in spindle.conf format - const std::map &options = getOptionsByID(); + if (!output_file.empty()) { + // Check if file exists + struct stat st; + if (stat(output_file.c_str(), &st) == 0 && !overwrite) { + std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; + std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; + return 1; + } + + // Open output file + output_fp = fopen(output_file.c_str(), "w"); + if (!output_fp) { + std::cerr << "Error: Could not open output file '" << output_file + << "': " << strerror(errno) << std::endl; + return 1; + } + + // Redirect stdout to file + saved_stdout = dup(STDOUT_FILENO); + dup2(fileno(output_fp), STDOUT_FILENO); + } + + // Generate config based on mode + if (all_mode) { + generate_all_config(confmap, mounts, header_warnings); + } else { + generate_default_config(confmap, mounts, header_warnings); + } - for (const auto &pair : options) { - format_config_value(confmap, pair.first, pair.second); + // Restore stdout and close file if we redirected + if (output_fp) { + fflush(stdout); + dup2(saved_stdout, STDOUT_FILENO); + close(saved_stdout); + fclose(output_fp); } return 0; From 7ca8393a083273555f5656116c9e368f586aece0 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 5 Nov 2026 13:33:37 -0800 Subject: [PATCH 08/18] Checkin prior to redesign. --- src/config_helper/man/spindle-config-helper.1 | 43 ++- src/config_helper/spindle-config-helper.cc | 340 ++++++++++++++---- 2 files changed, 305 insertions(+), 78 deletions(-) diff --git a/src/config_helper/man/spindle-config-helper.1 b/src/config_helper/man/spindle-config-helper.1 index 97afb0c9..837e5a73 100644 --- a/src/config_helper/man/spindle-config-helper.1 +++ b/src/config_helper/man/spindle-config-helper.1 @@ -52,13 +52,13 @@ Allow overwriting an existing output file when used with .BR \-o . .TP .BR \-c ", " \-\-check [=\fIFILE\fR] -Validate an existing -.I spindle.conf -file. Checks option names against the current Spindle version, validates -enum values, and reports unknown or deprecated options. If +Validate and merge Spindle configuration options. .I FILE -is not specified, only validates options passed after -.BR \-\- . +is optional. Reads options from config file (if provided) and/or +command-line (after +.BR \-\- ). +Command-line options override config file values. Validates option names +and values, outputting the merged and validated configuration. .TP .BR \-a ", " \-\-all Generate a complete configuration file including all Spindle options. @@ -110,21 +110,30 @@ The output includes warnings and recommendations as comments. .SS Check Mode When invoked with .BR \-c , -the tool validates an existing configuration file: -.IP \(bu 2 -Parses -.I key = value -pairs from the file +the tool validates and merges configuration options. The filename is optional, +allowing three use cases: +.TP +.B \-c FILE +Reads and validates options from +.IR FILE . +.TP +.B \-c \-\- \-\-option=value +Validates individual command-line options without reading a file. +Useful for testing whether an option exists and if its value is valid. +.TP +.B \-c FILE \-\- \-\-option=value +Reads options from +.IR FILE , +merges with command-line options (command-line wins), validates all options, +and outputs the merged configuration. +.PP +The tool validates: .IP \(bu 2 -Verifies option names are valid in the current Spindle version +Option names are valid in the current Spindle version .IP \(bu 2 -Validates enum values (e.g., level must be high/medium/low/off) +Enum values (e.g., level must be high/medium/low/off/default) .IP \(bu 2 Reports malformed lines, unknown options, and invalid values -.IP \(bu 2 -Command-line options passed after -.B \-\- -take precedence over config file values .SS All Mode When invoked with .BR \-a , diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 52bae440..bdf66e1f 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -774,6 +774,9 @@ static bool test_ssh_connectivity(const std::string& hostname, std::string& erro return false; } +// Forward declarations +static void print_config_header(std::vector& warnings); + // Parse a spindle.conf file and return key-value pairs // Returns: map of key -> value pairs // Ignores comments and blank lines @@ -837,6 +840,81 @@ static std::map parse_config_file(const std::string& f return config; } +// Parse command-line options from argv (options after --) +// Returns: map of key -> value pairs +static std::map parse_cmdline_options(int argc, char* argv[], + int separator_idx, + std::vector& warnings) +{ + std::map options; + + if (separator_idx == -1 || separator_idx + 1 >= argc) { + return options; // No options after -- + } + + // Parse options after -- + for (int i = separator_idx + 1; i < argc; i++) { + std::string arg = argv[i]; + + // Skip --usage + if (arg == "--usage") continue; + + // Parse --key=value or --key value + if (arg.substr(0, 2) == "--") { + std::string key, value; + size_t eq_pos = arg.find('='); + + if (eq_pos != std::string::npos) { + // --key=value + key = arg.substr(2, eq_pos - 2); + value = arg.substr(eq_pos + 1); + } else { + // --key (boolean or next arg is value) + key = arg.substr(2); + + // Check if next arg exists and doesn't start with -- + if (i + 1 < argc && argv[i + 1][0] != '-') { + value = argv[i + 1]; + i++; // Skip next arg + } else { + // Boolean flag + value = "true"; + } + } + + // Remove dashes from key to match config file format + key.erase(std::remove(key.begin(), key.end(), '-'), key.end()); + + if (!key.empty()) { + options[key] = value; + } + } else { + warnings.push_back("WARNING: Ignoring unrecognized argument: " + arg); + } + } + + return options; +} + +// Output a checked/validated config (for check mode) +static void output_checked_config(const std::map& config, + std::vector& warnings) +{ + // Print header + print_config_header(warnings); + + // Output all options in sorted order + std::vector keys; + for (const auto& kv : config) { + keys.push_back(kv.first); + } + std::sort(keys.begin(), keys.end()); + + for (const auto& key : keys) { + std::cout << key << " = " << config.at(key) << std::endl; + } +} + static void print_config_header(std::vector& warnings) { std::cout << "# spindle.conf generated by spindle-config-helper version " @@ -1409,12 +1487,16 @@ static void print_usage(const char *progname) << " * pythonprefix: Python installation directories\n" << " * localprefix: Directories excluded from caching\n" << " - Includes warnings and recommendations as comments\n\n" - << " Check mode (-c FILE):\n" - << " - Reads and parses existing spindle.conf file\n" - << " - Validates option names against current Spindle version\n" - << " - Validates enum values (e.g., level, security, launcher)\n" - << " - Reports unknown or deprecated options\n" - << " - Command-line options after -- override config file values\n\n" + << " Check mode (-c [FILE]):\n" + << " - FILE is optional: validates and outputs options from:\n" + << " * Config file (if FILE provided)\n" + << " * Command-line options after --\n" + << " - Command-line options override config file values\n" + << " - Validates option names and values\n" + << " - Use cases:\n" + << " * -c FILE : Validate file\n" + << " * -c -- --opt=val : Validate single option\n" + << " * -c FILE -- --opt : Validate file + override\n\n" << " All mode (-a):\n" << " - Enumerates all Spindle configuration options\n" << " - Comments out options with default values (prefix '#')\n" @@ -1542,6 +1624,9 @@ int main(int argc, char *argv[]) print_version(); return 0; } + if (strcmp(argv[i], "--usage") == 0) { + show_spindle_usage = true; + } if (strcmp(argv[i], "-o") == 0 && i + 1 < argc) { output_file = argv[i + 1]; i++; // Skip next argument @@ -1586,6 +1671,194 @@ int main(int argc, char *argv[]) } } + // Handle -- --usage early (don't generate config) + if (show_spindle_usage) { + // Pass through to Spindle's option system by continuing with normal flow + // but we'll build argv with --usage and let it print + // Actually, just build minimal argv and call gatherAllConfigInfo + // which will handle --usage and exit + + std::vector usage_argv; + usage_argv.push_back(argv[0]); + + // Add --usage flag + char *usage_flag = strdup("--usage"); + usage_argv.push_back(usage_flag); + + // Add options after -- (if any) + if (separator_idx != -1) { + for (int i = separator_idx + 1; i < argc; i++) { + // Skip --usage if it appears here too + if (strcmp(argv[i], "--usage") != 0) { + usage_argv.push_back(argv[i]); + } + } + } + + // Add dummy app + char *dummy_sep = strdup("--"); + char *dummy_app = strdup("/bin/true"); + usage_argv.push_back(dummy_sep); + usage_argv.push_back(dummy_app); + + char **usage_argv_ptr = usage_argv.data(); + int usage_argc = usage_argv.size(); + + ConfigMap confmap; + std::string errmsg; + gatherAllConfigInfo(usage_argc, usage_argv_ptr, false, confmap, errmsg); + + free(usage_flag); + free(dummy_sep); + free(dummy_app); + return 0; + } + + // Handle check mode with separate code path + if (check_mode) { + std::map config_values; + std::vector warnings; + + // Parse config file if provided + if (!check_file.empty()) { + config_values = parse_config_file(check_file, warnings); + } + + // Parse command-line options after -- + std::map cmdline_options = + parse_cmdline_options(argc, argv, separator_idx, warnings); + + // Merge: command-line overrides file + for (const auto& kv : cmdline_options) { + config_values[kv.first] = kv.second; + } + + // Validate all options by building argv and using gatherAllConfigInfo + // This reuses Spindle's validation without needing getOptionsByID + + std::vector val_argv; + std::vector val_strings; // Keep strings alive + + val_argv.push_back(argv[0]); // Program name + + // Build command-line arguments for validation + // Map config file keys (no dashes) to command-line format (with dashes) + // Use a static mapping to avoid iterating through option lists + std::map key_mapping = { + {"localprefix", "local-prefix"}, + {"pythonprefix", "python-prefix"}, + // Note: commpaths and cachepaths don't need mapping - they're single words + {"relocaout", "reloc-aout"}, + {"reloclibs", "reloc-libs"}, + {"relocpython", "reloc-python"}, + {"relocexec", "reloc-exec"}, + {"followfork", "follow-fork"}, + {"stopreloc", "stop-reloc"}, + {"pushpull", "push"}, + {"shmcachesize", "shmcache-size"}, + {"msgcachebuffer", "msgcache-buffer"}, + {"msgcachetimeout", "msgcache-timeout"}, + {"cleanupproc", "cleanup-proc"}, + {"cacheprefix", "cache-prefix"}, + {"hostbin", "host-bin"}, + {"startmultisession", "start-multisession"}, + {"startsession", "start-session"}, + {"endsession", "end-session"}, + {"runinsession", "run-in-session"}, + {"patchldso", "patch-ldso"}, + {"crashdedup", "crash-dedup"}, + {"crashaltstack", "crash-altstack"}, + {"crashlog", "crash-log"}, + {"rshcommand", "rsh-command"}, + {"enablersh", "enable-rsh"}, + {"execexcludes", "exec-excludes"}, + {"numaincludes", "numa-includes"}, + {"numaexcludes", "numa-excludes"}, + {"audittype", "audit-type"}, + {"disablelogging", "disable-logging"}, + {"noclean", "no-clean"}, + {"nohide", "no-hide"} + // Add more as needed - most single-word keys don't need mapping + }; + + for (const auto& kv : config_values) { + std::string cmdline_key = kv.first; + + // Check if we have a mapping for this key + auto it = key_mapping.find(kv.first); + if (it != key_mapping.end()) { + cmdline_key = it->second; + } + + // Add as two separate args: --key value + std::string opt = "--" + cmdline_key; + val_strings.push_back(opt); + val_strings.push_back(kv.second); + } + + for (auto& s : val_strings) { + val_argv.push_back(const_cast(s.c_str())); + } + + // Add dummy app + char *val_sep = strdup("--"); + char *val_app = strdup("/bin/true"); + val_argv.push_back(val_sep); + val_argv.push_back(val_app); + + char **val_argv_ptr = val_argv.data(); + int val_argc = val_argv.size(); + + ConfigMap val_confmap; + std::string val_errmsg; + if (!gatherAllConfigInfo(val_argc, val_argv_ptr, false, val_confmap, val_errmsg)) { + // Validation failed - gatherAllConfigInfo already printed error to stderr + // Do not generate config output + free(val_sep); + free(val_app); + return 1; + } + + free(val_sep); + free(val_app); + + // Handle file output if specified + FILE* output_fp = nullptr; + int saved_stdout = -1; + + if (!output_file.empty()) { + struct stat st; + if (stat(output_file.c_str(), &st) == 0 && !overwrite) { + std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; + std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; + return 1; + } + + output_fp = fopen(output_file.c_str(), "w"); + if (!output_fp) { + std::cerr << "Error: Could not open output file '" << output_file + << "': " << strerror(errno) << std::endl; + return 1; + } + + saved_stdout = dup(STDOUT_FILENO); + dup2(fileno(output_fp), STDOUT_FILENO); + } + + // Output the validated config + output_checked_config(config_values, warnings); + + // Restore stdout and close file if we redirected + if (output_fp) { + fflush(stdout); + dup2(saved_stdout, STDOUT_FILENO); + close(saved_stdout); + fclose(output_fp); + } + + return 0; + } + // Reconstruct argv to move Spindle options before -- and add dummy app // Input: spindle-config-helper [config-helper-opts] -- [spindle-opts] // Output: spindle-config-helper [spindle-opts] -- /bin/true (filtering out config-helper opts) @@ -1643,15 +1916,6 @@ int main(int argc, char *argv[]) new_argv.push_back(dummy_sep); new_argv.push_back(dummy_app); - // Handle check mode: parse config file if provided - std::map check_config; - std::vector check_warnings; - - if (check_mode && !check_file.empty()) { - check_config = parse_config_file(check_file, check_warnings); - // Config values will be merged with confmap after parsing command-line options - } - char **modified_argv = new_argv.data(); int modified_argc = new_argv.size(); @@ -1671,58 +1935,12 @@ int main(int argc, char *argv[]) free(dummy_app); free(dummy_sep); - // In check mode, merge config file values with confmap - // Command-line options take precedence (they're already in confmap) - if (check_mode && !check_config.empty()) { - // For each config file option, check if it's already set - // If not set, add it to confmap - for (const auto& kv : check_config) { - const std::string& key = kv.first; - const std::string& value = kv.second; - - // Find the corresponding SpindleConfigID - const std::map& all_opts = getOptionsByID(); - bool found = false; - SpindleConfigID found_id; - - for (const auto& pair : all_opts) { - const SpindleOption& opt = pair.second; - if (opt.cmdline_long) { - // Remove dashes from cmdline_long to match config file format - std::string opt_name = opt.cmdline_long; - opt_name.erase(std::remove(opt_name.begin(), opt_name.end(), '-'), opt_name.end()); - if (opt_name == key) { - found_id = opt.config_id; - found = true; - break; - } - } - } - - if (!found) { - check_warnings.push_back("WARNING: Unknown option in config file: " + key); - continue; - } - - // Validate the value from config file - // Note: Full merging with command-line overrides needs deeper integration - // For now, we just validate and report warnings - std::string errstring; - if (!confmap.set(found_id, value, errstring)) { - check_warnings.push_back("WARNING: Invalid value for " + key + " in config file: " + errstring); - } - } - } - // Parse and analyze mount points std::vector mounts = parse_mounts(); // Collect header warnings (will add $TMPDIR and SSH checks later) std::vector header_warnings; - // Add any warnings from config file parsing - header_warnings.insert(header_warnings.end(), check_warnings.begin(), check_warnings.end()); - // Handle output file if specified FILE* output_fp = nullptr; int saved_stdout = -1; From 7be2b23d7182b58aa023aeac4e88b98e3bda5a14 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 5 Nov 2026 21:05:33 -0800 Subject: [PATCH 09/18] [config_helper] Rewrite phases 1-3. --- src/config_helper/Makefile.am | 7 +- src/config_helper/Makefile.in | 19 +- src/config_helper/spindle-config-helper.cc | 322 +++++++++++++++++---- 3 files changed, 281 insertions(+), 67 deletions(-) diff --git a/src/config_helper/Makefile.am b/src/config_helper/Makefile.am index 45ba4128..6b5ea773 100644 --- a/src/config_helper/Makefile.am +++ b/src/config_helper/Makefile.am @@ -10,7 +10,12 @@ spindle_config_helper_CPPFLAGS = -I$(top_srcdir)/src/include \ -I$(top_srcdir)/src/server/comlib \ -I$(top_srcdir)/src/cobo \ -I$(top_srcdir)/src/fe/comlib \ - -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" + -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" \ + $(MPI_CXXFLAGS) + +spindle_config_helper_CXXFLAGS = $(MPI_CXXFLAGS) + +spindle_config_helper_LDFLAGS = $(MPI_CXXLDFLAGS) spindle_config_helper_LDADD = $(top_builddir)/src/fe/startup/libconfigparse.a \ $(top_builddir)/src/fe/logging/libspindleflogc.la \ diff --git a/src/config_helper/Makefile.in b/src/config_helper/Makefile.in index 08d47f8c..ce59e413 100644 --- a/src/config_helper/Makefile.in +++ b/src/config_helper/Makefile.in @@ -118,6 +118,10 @@ AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) am__v_lt_0 = --silent am__v_lt_1 = +spindle_config_helper_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CXXLD) \ + $(spindle_config_helper_CXXFLAGS) $(CXXFLAGS) \ + $(spindle_config_helper_LDFLAGS) $(LDFLAGS) -o $@ AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) am__v_P_0 = false @@ -382,8 +386,11 @@ spindle_config_helper_CPPFLAGS = -I$(top_srcdir)/src/include \ -I$(top_srcdir)/src/server/comlib \ -I$(top_srcdir)/src/cobo \ -I$(top_srcdir)/src/fe/comlib \ - -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" + -DPKGSYSCONFDIR=\"$(PKGSYSCONF_DIR)\" \ + $(MPI_CXXFLAGS) +spindle_config_helper_CXXFLAGS = $(MPI_CXXFLAGS) +spindle_config_helper_LDFLAGS = $(MPI_CXXLDFLAGS) spindle_config_helper_LDADD = $(top_builddir)/src/fe/startup/libconfigparse.a \ $(top_builddir)/src/fe/logging/libspindleflogc.la \ -lpthread @@ -474,7 +481,7 @@ clean-binPROGRAMS: spindle-config-helper$(EXEEXT): $(spindle_config_helper_OBJECTS) $(spindle_config_helper_DEPENDENCIES) $(EXTRA_spindle_config_helper_DEPENDENCIES) @rm -f spindle-config-helper$(EXEEXT) - $(AM_V_CXXLD)$(CXXLINK) $(spindle_config_helper_OBJECTS) $(spindle_config_helper_LDADD) $(LIBS) + $(AM_V_CXXLD)$(spindle_config_helper_LINK) $(spindle_config_helper_OBJECTS) $(spindle_config_helper_LDADD) $(LIBS) mostlyclean-compile: -rm -f *.$(OBJEXT) @@ -515,18 +522,18 @@ am--depfiles: $(am__depfiles_remade) @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< spindle_config_helper-spindle-config-helper.o: spindle-config-helper.cc -@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_config_helper-spindle-config-helper.o -MD -MP -MF $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo -c -o spindle_config_helper-spindle-config-helper.o `test -f 'spindle-config-helper.cc' || echo '$(srcdir)/'`spindle-config-helper.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(spindle_config_helper_CXXFLAGS) $(CXXFLAGS) -MT spindle_config_helper-spindle-config-helper.o -MD -MP -MF $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo -c -o spindle_config_helper-spindle-config-helper.o `test -f 'spindle-config-helper.cc' || echo '$(srcdir)/'`spindle-config-helper.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo $(DEPDIR)/spindle_config_helper-spindle-config-helper.Po @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='spindle-config-helper.cc' object='spindle_config_helper-spindle-config-helper.o' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_config_helper-spindle-config-helper.o `test -f 'spindle-config-helper.cc' || echo '$(srcdir)/'`spindle-config-helper.cc +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(spindle_config_helper_CXXFLAGS) $(CXXFLAGS) -c -o spindle_config_helper-spindle-config-helper.o `test -f 'spindle-config-helper.cc' || echo '$(srcdir)/'`spindle-config-helper.cc spindle_config_helper-spindle-config-helper.obj: spindle-config-helper.cc -@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_config_helper-spindle-config-helper.obj -MD -MP -MF $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo -c -o spindle_config_helper-spindle-config-helper.obj `if test -f 'spindle-config-helper.cc'; then $(CYGPATH_W) 'spindle-config-helper.cc'; else $(CYGPATH_W) '$(srcdir)/spindle-config-helper.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(spindle_config_helper_CXXFLAGS) $(CXXFLAGS) -MT spindle_config_helper-spindle-config-helper.obj -MD -MP -MF $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo -c -o spindle_config_helper-spindle-config-helper.obj `if test -f 'spindle-config-helper.cc'; then $(CYGPATH_W) 'spindle-config-helper.cc'; else $(CYGPATH_W) '$(srcdir)/spindle-config-helper.cc'; fi` @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_config_helper-spindle-config-helper.Tpo $(DEPDIR)/spindle_config_helper-spindle-config-helper.Po @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='spindle-config-helper.cc' object='spindle_config_helper-spindle-config-helper.obj' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_config_helper-spindle-config-helper.obj `if test -f 'spindle-config-helper.cc'; then $(CYGPATH_W) 'spindle-config-helper.cc'; else $(CYGPATH_W) '$(srcdir)/spindle-config-helper.cc'; fi` +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_config_helper_CPPFLAGS) $(CPPFLAGS) $(spindle_config_helper_CXXFLAGS) $(CXXFLAGS) -c -o spindle_config_helper-spindle-config-helper.obj `if test -f 'spindle-config-helper.cc'; then $(CYGPATH_W) 'spindle-config-helper.cc'; else $(CYGPATH_W) '$(srcdir)/spindle-config-helper.cc'; fi` mostlyclean-libtool: -rm -f *.lo diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index bdf66e1f..0558578f 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -34,6 +34,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include extern "C" { #include "keyfile.h" @@ -568,22 +569,28 @@ static bool validate_tmpdir_path(const std::string& path, static std::string determine_best_path(const std::vector& mounts, std::vector& warnings) { - // Check $TMPDIR first - const char* tmpdir_env = getenv("TMPDIR"); + // Priority order: TMPDIR, TEMPDIR, TMP, then /tmp + const char* env_vars[] = {"TMPDIR", "TEMPDIR", "TMP", nullptr}; - if (tmpdir_env && tmpdir_env[0] != '\0') { - std::string tmpdir = tmpdir_env; - std::string error_msg; + for (int i = 0; env_vars[i] != nullptr; i++) { + const char* env_value = getenv(env_vars[i]); - if (validate_tmpdir_path(tmpdir, mounts, error_msg)) { - return tmpdir; - } else { - warnings.push_back("$TMPDIR is set to '" + tmpdir + "' but not usable: " + error_msg); + if (env_value && env_value[0] != '\0') { + std::string tmpdir = env_value; + std::string error_msg; + + if (validate_tmpdir_path(tmpdir, mounts, error_msg)) { + return tmpdir; + } else { + warnings.push_back(std::string("$") + env_vars[i] + " is set to '" + + tmpdir + "' but not usable: " + error_msg); + } } - } else { - warnings.push_back("$TMPDIR is not set"); } + // None of the environment variables are set or usable + warnings.push_back("$TMPDIR, $TEMPDIR, and $TMP are not set or not usable"); + // Fall back to /tmp std::string error_msg; if (validate_tmpdir_path("/tmp", mounts, error_msg)) { @@ -629,17 +636,88 @@ static std::string get_python_prefix_from_which(const char* python_cmd) return result; } +// Get Python paths from sys.prefix and sys.path +static std::vector get_python_sys_paths() +{ + std::set paths; + + // Run Python to get sys.prefix and sys.path + const char* cmd = "python3 -c \"import sys; print(sys.prefix); print('\\n'.join(sys.path))\" 2>/dev/null"; + FILE* fp = popen(cmd, "r"); + if (!fp) { + return {}; + } + + char line[PATH_MAX]; + while (fgets(line, sizeof(line), fp) != nullptr) { + // Remove trailing newline + size_t len = strlen(line); + if (len > 0 && line[len-1] == '\n') { + line[len-1] = '\0'; + } + + if (strlen(line) > 0) { + // Check if this looks like a valid path + if (line[0] == '/' && access(line, F_OK) == 0) { + paths.insert(line); + } + } + } + pclose(fp); + + return std::vector(paths.begin(), paths.end()); +} + static std::vector discover_python_prefixes() { std::set prefixes; - // Try 'which python' + // Priority 1: Check PYTHONPREFIX environment variable + const char* pythonprefix_env = getenv("PYTHONPREFIX"); + if (pythonprefix_env && pythonprefix_env[0] != '\0') { + // Parse colon-separated list + std::string paths_str(pythonprefix_env); + size_t start = 0; + size_t end = paths_str.find(':'); + + while (end != std::string::npos) { + std::string path = paths_str.substr(start, end - start); + if (!path.empty() && access(path.c_str(), F_OK) == 0) { + prefixes.insert(path); + } + start = end + 1; + end = paths_str.find(':', start); + } + + // Add last path + std::string path = paths_str.substr(start); + if (!path.empty() && access(path.c_str(), F_OK) == 0) { + prefixes.insert(path); + } + + // If PYTHONPREFIX provided valid paths, use only those + if (!prefixes.empty()) { + return std::vector(prefixes.begin(), prefixes.end()); + } + } + + // Priority 2: Try Python sys.prefix and sys.path + std::vector sys_paths = get_python_sys_paths(); + if (!sys_paths.empty()) { + for (const auto& path : sys_paths) { + prefixes.insert(path); + } + if (!prefixes.empty()) { + return std::vector(prefixes.begin(), prefixes.end()); + } + } + + // Priority 3: Fall back to 'which python' discovery std::string py2 = get_python_prefix_from_which("python"); if (!py2.empty()) { prefixes.insert(py2); } - // Try 'which python3' std::string py3 = get_python_prefix_from_which("python3"); if (!py3.empty()) { prefixes.insert(py3); @@ -723,6 +801,97 @@ static std::string get_hostname_for_ssh() return std::string(hostname); } +// Forward declaration +static bool test_ssh_connectivity(const std::string& hostname, std::string& error_msg); + +// Structure to hold SSH test results for MPI communication +struct SSHTestResult { + int rank; + int target_rank; + bool success; + char error_msg[256]; + char source_host[256]; + char target_host[256]; +}; + +// Ring topology SSH test: each rank tests SSH to next rank +static void test_ssh_ring(int mpi_rank, int mpi_size, std::vector& warnings) +{ + // Get local hostname + std::string local_hostname = get_hostname_for_ssh(); + + // Exchange hostnames among all ranks using MPI_Allgather + const int MAX_HOSTNAME_LEN = 256; + char local_host[MAX_HOSTNAME_LEN]; + char all_hosts[MAX_HOSTNAME_LEN * mpi_size]; + + strncpy(local_host, local_hostname.c_str(), MAX_HOSTNAME_LEN - 1); + local_host[MAX_HOSTNAME_LEN - 1] = '\0'; + + MPI_Allgather(local_host, MAX_HOSTNAME_LEN, MPI_CHAR, + all_hosts, MAX_HOSTNAME_LEN, MPI_CHAR, + MPI_COMM_WORLD); + + // Determine target: (rank + 1) % size + int target_rank = (mpi_rank + 1) % mpi_size; + std::string target_hostname(&all_hosts[target_rank * MAX_HOSTNAME_LEN]); + + // Test SSH to target + SSHTestResult result; + result.rank = mpi_rank; + result.target_rank = target_rank; + strncpy(result.source_host, local_hostname.c_str(), 255); + result.source_host[255] = '\0'; + strncpy(result.target_host, target_hostname.c_str(), 255); + result.target_host[255] = '\0'; + + std::string ssh_error; + result.success = test_ssh_connectivity(target_hostname, ssh_error); + strncpy(result.error_msg, ssh_error.c_str(), 255); + result.error_msg[255] = '\0'; + + // Gather all results to rank 0 + SSHTestResult all_results[mpi_size]; + MPI_Gather(&result, sizeof(SSHTestResult), MPI_BYTE, + all_results, sizeof(SSHTestResult), MPI_BYTE, + 0, MPI_COMM_WORLD); + + // Rank 0 processes and reports results + if (mpi_rank == 0) { + int successful_tests = 0; + bool all_passed = true; + + for (int i = 0; i < mpi_size; i++) { + if (all_results[i].success) { + successful_tests++; + } else { + all_passed = false; + } + } + + if (all_passed) { + char msg[512]; + snprintf(msg, sizeof(msg), "SSH ring test passed: All %d ranks successfully tested SSH connectivity", mpi_size); + warnings.push_back(std::string(msg)); + } else { + char msg[512]; + snprintf(msg, sizeof(msg), "SSH ring test: %d/%d successful", successful_tests, mpi_size); + warnings.push_back(std::string(msg)); + + // Report failures + for (int i = 0; i < mpi_size; i++) { + if (!all_results[i].success) { + snprintf(msg, sizeof(msg), " Rank %d (%s) -> Rank %d (%s): FAILED - %s", + all_results[i].rank, all_results[i].source_host, + all_results[i].target_rank, all_results[i].target_host, + all_results[i].error_msg); + warnings.push_back(std::string(msg)); + } + } + } + } +} + static bool test_ssh_connectivity(const std::string& hostname, std::string& error_msg) { if (hostname.empty()) { @@ -961,7 +1130,8 @@ static void print_path_variable(const std::string& name, const std::string& valu static void generate_default_config(const ConfigMap& confmap, const std::vector& mounts, - std::vector& header_warnings) + std::vector& header_warnings, + int mpi_rank, int mpi_size) { // Determine best path for commpaths/cachepaths if using defaults std::string recommended_path; @@ -993,15 +1163,7 @@ static void generate_default_config(const ConfigMap& confmap, } } - // Test SSH connectivity - std::string hostname = get_hostname_for_ssh(); - std::string ssh_error; - if (!test_ssh_connectivity(hostname, ssh_error)) { - header_warnings.push_back("SSH test failed for " + hostname + ": " + ssh_error); - header_warnings.push_back(" (SSH keys may not be configured for compute node access)"); - } else { - header_warnings.push_back("SSH test passed for " + hostname); - } + // SSH test results already collected in header_warnings (done in main before calling this function) // Header print_config_header(header_warnings); @@ -1597,6 +1759,17 @@ static void format_config_value(const ConfigMap &confmap, SpindleConfigID id, int main(int argc, char *argv[]) { + // Initialize MPI + int mpi_rank = 0, mpi_size = 1; + int mpi_initialized = 0; + MPI_Initialized(&mpi_initialized); + + if (!mpi_initialized) { + MPI_Init(&argc, &argv); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + // Handle -h/--help and -V/--version before config parsing // Also check for -- --usage which should show Spindle's usage bool show_spindle_usage = false; @@ -1617,11 +1790,17 @@ int main(int argc, char *argv[]) if (separator_idx == -1) { // Before --: check for config-helper options if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - print_usage(argv[0]); + if (mpi_rank == 0) { + print_usage(argv[0]); + } + MPI_Finalize(); return 0; } if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) { - print_version(); + if (mpi_rank == 0) { + print_version(); + } + MPI_Finalize(); return 0; } if (strcmp(argv[i], "--usage") == 0) { @@ -1711,6 +1890,7 @@ int main(int argc, char *argv[]) free(usage_flag); free(dummy_sep); free(dummy_app); + MPI_Finalize(); return 0; } @@ -1856,6 +2036,7 @@ int main(int argc, char *argv[]) fclose(output_fp); } + MPI_Finalize(); return 0; } @@ -1935,52 +2116,73 @@ int main(int argc, char *argv[]) free(dummy_app); free(dummy_sep); - // Parse and analyze mount points + // Parse and analyze mount points (all ranks need this for SSH test) std::vector mounts = parse_mounts(); // Collect header warnings (will add $TMPDIR and SSH checks later) std::vector header_warnings; - // Handle output file if specified - FILE* output_fp = nullptr; - int saved_stdout = -1; - - if (!output_file.empty()) { - // Check if file exists - struct stat st; - if (stat(output_file.c_str(), &st) == 0 && !overwrite) { - std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; - std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; - return 1; + // ALL RANKS: Test SSH connectivity using ring topology (uses collective MPI operations) + if (mpi_size > 1) { + test_ssh_ring(mpi_rank, mpi_size, header_warnings); + } else if (mpi_rank == 0) { + // Single node - test local SSH only + std::string hostname = get_hostname_for_ssh(); + std::string ssh_error; + if (!test_ssh_connectivity(hostname, ssh_error)) { + header_warnings.push_back("SSH test failed for " + hostname + ": " + ssh_error); + header_warnings.push_back(" (SSH keys may not be configured for compute node access)"); + } else { + header_warnings.push_back("SSH test passed for " + hostname); } + } - // Open output file - output_fp = fopen(output_file.c_str(), "w"); - if (!output_fp) { - std::cerr << "Error: Could not open output file '" << output_file - << "': " << strerror(errno) << std::endl; - return 1; - } + // Only rank 0 generates and prints output + if (mpi_rank == 0) { + // Handle output file if specified + FILE* output_fp = nullptr; + int saved_stdout = -1; - // Redirect stdout to file - saved_stdout = dup(STDOUT_FILENO); - dup2(fileno(output_fp), STDOUT_FILENO); - } + if (!output_file.empty()) { + // Check if file exists + struct stat st; + if (stat(output_file.c_str(), &st) == 0 && !overwrite) { + std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; + std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; + MPI_Finalize(); + return 1; + } - // Generate config based on mode - if (all_mode) { - generate_all_config(confmap, mounts, header_warnings); - } else { - generate_default_config(confmap, mounts, header_warnings); - } + // Open output file + output_fp = fopen(output_file.c_str(), "w"); + if (!output_fp) { + std::cerr << "Error: Could not open output file '" << output_file + << "': " << strerror(errno) << std::endl; + MPI_Finalize(); + return 1; + } + + // Redirect stdout to file + saved_stdout = dup(STDOUT_FILENO); + dup2(fileno(output_fp), STDOUT_FILENO); + } + + // Generate config based on mode + if (all_mode) { + generate_all_config(confmap, mounts, header_warnings); + } else { + generate_default_config(confmap, mounts, header_warnings, mpi_rank, mpi_size); + } - // Restore stdout and close file if we redirected - if (output_fp) { - fflush(stdout); - dup2(saved_stdout, STDOUT_FILENO); - close(saved_stdout); - fclose(output_fp); + // Restore stdout and close file if we redirected + if (output_fp) { + fflush(stdout); + dup2(saved_stdout, STDOUT_FILENO); + close(saved_stdout); + fclose(output_fp); + } } + MPI_Finalize(); return 0; } From 6ce8b6f79f190a8ecfbcaa1b8cb124b1afc0e21b Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 5 Nov 2026 21:10:40 -0800 Subject: [PATCH 10/18] [config_helper] Phase 4. --- src/config_helper/spindle-config-helper.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 0558578f..0db1d26d 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -55,6 +55,7 @@ static bool is_remote_filesystem(const std::string& fstype) static const char* remote_types[] = { "nfs", "nfs4", "lustre", "gpfs", "panfs", "ceph", "glusterfs", "afs", "smbfs", "cifs", + "overlay", // Overlay filesystems are considered non-local nullptr }; From 3e1fa8b5e7ac7b390eb266526025f22b345b7a99 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Thu, 5 Nov 2026 22:57:01 -0800 Subject: [PATCH 11/18] [config-helper] Phase 5 --- src/config_helper/spindle-config-helper.cc | 359 ++++++++++++++------- 1 file changed, 243 insertions(+), 116 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 0db1d26d..685fbb69 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -572,32 +572,44 @@ static std::string determine_best_path(const std::vector& mounts, { // Priority order: TMPDIR, TEMPDIR, TMP, then /tmp const char* env_vars[] = {"TMPDIR", "TEMPDIR", "TMP", nullptr}; + bool any_env_set = false; + std::string first_valid_path; + // Check environment variables for (int i = 0; env_vars[i] != nullptr; i++) { const char* env_value = getenv(env_vars[i]); if (env_value && env_value[0] != '\0') { + any_env_set = true; std::string tmpdir = env_value; std::string error_msg; if (validate_tmpdir_path(tmpdir, mounts, error_msg)) { - return tmpdir; - } else { - warnings.push_back(std::string("$") + env_vars[i] + " is set to '" + - tmpdir + "' but not usable: " + error_msg); + if (first_valid_path.empty()) { + first_valid_path = tmpdir; + } } } } - // None of the environment variables are set or usable - warnings.push_back("$TMPDIR, $TEMPDIR, and $TMP are not set or not usable"); + // Generate appropriate WARNING or Note for TMPDIR section + if (!any_env_set) { + warnings.push_back("WARNING: The TMPDIR environment variable was not set."); + } else if (!first_valid_path.empty()) { + warnings.push_back("Note: The TMPDIR path is local and writeable."); + } else { + warnings.push_back("WARNING: The TMPDIR environment variable is not a local, writeable filesystem."); + } + + // Return best path found, or fall back to /tmp + if (!first_valid_path.empty()) { + return first_valid_path; + } // Fall back to /tmp std::string error_msg; if (validate_tmpdir_path("/tmp", mounts, error_msg)) { return "/tmp"; - } else { - warnings.push_back("/tmp is not usable: " + error_msg); } // No suitable path found @@ -642,8 +654,9 @@ static std::vector get_python_sys_paths() { std::set paths; - // Run Python to get sys.prefix and sys.path - const char* cmd = "python3 -c \"import sys; print(sys.prefix); print('\\n'.join(sys.path))\" 2>/dev/null"; + // Run Python with PYTHONHOME unset to get system defaults + // This ensures we get valid paths even if PYTHONHOME is broken + const char* cmd = "unset PYTHONHOME; python3 -c \"import sys; print(sys.prefix); print('\\n'.join(sys.path))\" 2>/dev/null"; FILE* fp = popen(cmd, "r"); if (!fp) { return {}; @@ -669,21 +682,91 @@ static std::vector get_python_sys_paths() return std::vector(paths.begin(), paths.end()); } +// Deduplicate paths by keeping only top-level directories +// If /foo is in the list and /foo/bar is also in the list, keep only /foo +static std::vector deduplicate_paths(const std::vector& paths) +{ + if (paths.empty()) return paths; + + // Sort by path length (shortest first) + std::vector sorted_paths = paths; + std::sort(sorted_paths.begin(), sorted_paths.end(), + [](const std::string& a, const std::string& b) { + return a.length() < b.length(); + }); + + std::vector result; + for (const auto& path : sorted_paths) { + bool is_subpath = false; + + // Check if this path is under any already-kept path + for (const auto& kept : result) { + // Check if path starts with kept/ + if (path.size() > kept.size() && + path.compare(0, kept.size(), kept) == 0 && + (path[kept.size()] == '/' || kept == "/")) { + is_subpath = true; + break; + } + } + + if (!is_subpath) { + result.push_back(path); + } + } + + return result; +} + +// Check if a path contains Python libraries +static bool is_valid_python_prefix(const std::string& path) +{ + // Check for lib/python* or lib64/python* directories + std::string lib_path = path + "/lib"; + std::string lib64_path = path + "/lib64"; + + DIR* dir = opendir(lib_path.c_str()); + if (dir) { + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strncmp(entry->d_name, "python", 6) == 0) { + closedir(dir); + return true; + } + } + closedir(dir); + } + + dir = opendir(lib64_path.c_str()); + if (dir) { + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strncmp(entry->d_name, "python", 6) == 0) { + closedir(dir); + return true; + } + } + closedir(dir); + } + + return false; +} + static std::vector discover_python_prefixes() { std::set prefixes; - // Priority 1: Check PYTHONPREFIX environment variable - const char* pythonprefix_env = getenv("PYTHONPREFIX"); - if (pythonprefix_env && pythonprefix_env[0] != '\0') { + // Priority 1: Check PYTHONHOME environment variable and validate paths + const char* pythonhome_env = getenv("PYTHONHOME"); + if (pythonhome_env && pythonhome_env[0] != '\0') { // Parse colon-separated list - std::string paths_str(pythonprefix_env); + std::string paths_str(pythonhome_env); size_t start = 0; size_t end = paths_str.find(':'); while (end != std::string::npos) { std::string path = paths_str.substr(start, end - start); - if (!path.empty() && access(path.c_str(), F_OK) == 0) { + if (!path.empty() && is_valid_python_prefix(path)) { prefixes.insert(path); } start = end + 1; @@ -692,39 +775,25 @@ static std::vector discover_python_prefixes() // Add last path std::string path = paths_str.substr(start); - if (!path.empty() && access(path.c_str(), F_OK) == 0) { + if (!path.empty() && is_valid_python_prefix(path)) { prefixes.insert(path); } - // If PYTHONPREFIX provided valid paths, use only those + // If PYTHONHOME provided valid Python installations, use only those if (!prefixes.empty()) { - return std::vector(prefixes.begin(), prefixes.end()); + std::vector all_paths(prefixes.begin(), prefixes.end()); + return deduplicate_paths(all_paths); } } - // Priority 2: Try Python sys.prefix and sys.path + // Priority 2: Fall back to Python sys.prefix (if PYTHONHOME not set or had no valid paths) std::vector sys_paths = get_python_sys_paths(); - if (!sys_paths.empty()) { - for (const auto& path : sys_paths) { - prefixes.insert(path); - } - if (!prefixes.empty()) { - return std::vector(prefixes.begin(), prefixes.end()); - } - } - - // Priority 3: Fall back to 'which python' discovery - std::string py2 = get_python_prefix_from_which("python"); - if (!py2.empty()) { - prefixes.insert(py2); - } - - std::string py3 = get_python_prefix_from_which("python3"); - if (!py3.empty()) { - prefixes.insert(py3); + for (const auto& path : sys_paths) { + prefixes.insert(path); } - return std::vector(prefixes.begin(), prefixes.end()); + std::vector all_paths(prefixes.begin(), prefixes.end()); + return deduplicate_paths(all_paths); } static bool has_remote_mount_under(const std::string& prefix, const std::vector& mounts) @@ -872,17 +941,16 @@ static void test_ssh_ring(int mpi_rank, int mpi_size, std::vector& if (all_passed) { char msg[512]; - snprintf(msg, sizeof(msg), "SSH ring test passed: All %d ranks successfully tested SSH connectivity", mpi_size); + snprintf(msg, sizeof(msg), "Note: Tested ssh successfully under MPI using %d nodes.", mpi_size); warnings.push_back(std::string(msg)); } else { - char msg[512]; - snprintf(msg, sizeof(msg), "SSH ring test: %d/%d successful", successful_tests, mpi_size); - warnings.push_back(std::string(msg)); + warnings.push_back("WARNING: Executing under MPI, but ssh communication among compute nodes failed."); - // Report failures + // Report detailed failures as additional lines + char msg[512]; for (int i = 0; i < mpi_size; i++) { if (!all_results[i].success) { - snprintf(msg, sizeof(msg), " Rank %d (%s) -> Rank %d (%s): FAILED - %s", + snprintf(msg, sizeof(msg), "# Rank %d (%s) -> Rank %d (%s): FAILED - %s", all_results[i].rank, all_results[i].source_host, all_results[i].target_rank, all_results[i].target_host, all_results[i].error_msg); @@ -1092,24 +1160,12 @@ static void print_config_header(std::vector& warnings) std::cout << "# Generated: " << get_timestamp() << std::endl; std::cout << "#" << std::endl; - if (!warnings.empty()) { - // Check if we have any actual warnings vs just informational notes - bool has_warnings = false; - for (const auto& msg : warnings) { - if (msg.find("WARNING") != std::string::npos || - msg.find("CRITICAL") != std::string::npos || - msg.find("failed") != std::string::npos || - msg.find("not set") != std::string::npos || - msg.find("not usable") != std::string::npos) { - has_warnings = true; - break; - } - } + // Print each warning/note on its own line (already prefixed with # WARNING: or # Note:) + for (const auto& warning : warnings) { + std::cout << "# " << warning << std::endl; + } - std::cout << "# " << (has_warnings ? "WARNINGS:" : "NOTES:") << std::endl; - for (const auto& warning : warnings) { - std::cout << "# " << warning << std::endl; - } + if (!warnings.empty()) { std::cout << "#" << std::endl; } } @@ -1122,11 +1178,13 @@ static void print_path_variable(const std::string& name, const std::string& valu std::cout << "# " << comment << std::endl; } - std::cout << name << " = " << value << std::endl; - - if (!comments.empty()) { - std::cout << std::endl; // Blank line after commented variables + // If value is empty, comment out the line + if (value.empty()) { + std::cout << "# " << name << " =" << std::endl; + } else { + std::cout << name << " = " << value << std::endl; } + // No blank line after - we put blank lines BEFORE sections instead } static void generate_default_config(const ConfigMap& confmap, @@ -1277,26 +1335,7 @@ static void generate_default_config(const ConfigMap& confmap, } } - // Build comments for bad paths (these came from configure/defaults, not user choice) - if (!bad_paths.empty()) { - if (check_python) { - comments.push_back("NOTE: The following paths are not valid Python installations:"); - } else { - comments.push_back("NOTE: The following paths from configure/defaults are not usable:"); - } - for (size_t i = 0; i < path_list.size(); i++) { - bool is_bad = (std::find(bad_paths.begin(), bad_paths.end(), path_list[i]) != bad_paths.end()); - if (is_bad) { - comments.push_back(" " + path_list[i] + ":"); - for (const auto& warning : analyses[i].warnings) { - comments.push_back(" " + warning); - } - } - } - if (!good_paths.empty()) { - comments.push_back(" (These paths have been filtered out)"); - } - } + // Silently drop bad paths - no need to warn about them // Use only good paths in output if (!good_paths.empty()) { @@ -1307,13 +1346,11 @@ static void generate_default_config(const ConfigMap& confmap, } paths = filtered; } else { - // No good paths at all + // No good paths at all - output commented line + paths = ""; if (check_python) { - // For pythonprefix, skip output if no valid Python found comments.push_back("WARNING: No valid Python installations found"); - return; } - paths = ""; } // Add comment for usable paths if no issues @@ -1331,8 +1368,24 @@ static void generate_default_config(const ConfigMap& confmap, print_path_variable(name, paths, comments); }; + // Add comment block for commpaths and cachepaths + std::cout << "#" << std::endl; + std::cout << "# commpaths and cachepaths" << std::endl; + std::cout << "# These two settings are colon-separated lists of local (on-node) storage that can be used for Spindle" << std::endl; + std::cout << "# sockets and caches, respectively. If the entries below are commented out, they either pointed to" << std::endl; + std::cout << "# a non-local filesystem or were not writeable by this user. Both of these are based on the" << std::endl; + std::cout << "# TMPDIR environment variable; fallback is /tmp." << std::endl; output_path_option(confCommPaths, "commpaths"); output_path_option(confCachePaths, "cachepaths"); + + // Add comment block for pythonpath + std::cout << "#" << std::endl; + std::cout << "# pythonpath" << std::endl; + std::cout << "# This is a colon-separated list of paths that contain python packages (including site-specific" << std::endl; + std::cout << "# packages). The directories will include PYTHONHOME (if specified) and the results of" << std::endl; + std::cout << "# checking python's sys.prefix and sys.path. If the entry is commented out, the program could" << std::endl; + std::cout << "# not locate python libraries." << std::endl; + output_path_option(confPythonPrefix, "pythonprefix", true); // Special handling for localprefix - generate from top-level dirs without remote mounts @@ -1386,9 +1439,12 @@ static void generate_default_config(const ConfigMap& confmap, output_value += (dir == "/" ? "/" : dir + "/"); } - if (comments.empty()) { - comments.push_back("localprefix: Top-level directories without remote mounts"); - } + // Add comment block for localprefix + std::cout << "#" << std::endl; + std::cout << "# localprefix" << std::endl; + std::cout << "# This is a colon-separated list of paths that Spindle should treat as node-local filesystems" << std::endl; + std::cout << "# (i.e., Spindle will not attempt to cache anything underneath these directories). Overlay" << std::endl; + std::cout << "# filesystems are assumed to be non-local." << std::endl; print_path_variable("localprefix", output_value, comments); } @@ -1758,18 +1814,48 @@ static void format_config_value(const ConfigMap &confmap, SpindleConfigID id, } } +// Detect if we're running under an MPI launcher by checking for MPI-related environment variables +static bool running_under_mpi() +{ + // Check for common MPI launcher environment variables + const char* mpi_env_vars[] = { + "PMI_RANK", // PMI-based launchers + "PMI_SIZE", + "PMIX_RANK", // PMIx + "SLURM_PROCID", // Slurm + "SLURM_NTASKS", + "FLUX_TASK_RANK", // Flux + "FLUX_JOB_SIZE", + "OMPI_COMM_WORLD_RANK", // OpenMPI + "MPI_LOCALRANKID", // Intel MPI + nullptr + }; + + for (int i = 0; mpi_env_vars[i] != nullptr; i++) { + if (getenv(mpi_env_vars[i]) != nullptr) { + return true; + } + } + + return false; +} + int main(int argc, char *argv[]) { - // Initialize MPI + // Initialize MPI if running under an MPI launcher int mpi_rank = 0, mpi_size = 1; + bool is_mpi_job = running_under_mpi(); int mpi_initialized = 0; - MPI_Initialized(&mpi_initialized); - if (!mpi_initialized) { - MPI_Init(&argc, &argv); + if (is_mpi_job) { + MPI_Initialized(&mpi_initialized); + + if (!mpi_initialized) { + MPI_Init(&argc, &argv); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); } - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); // Handle -h/--help and -V/--version before config parsing // Also check for -- --usage which should show Spindle's usage @@ -1794,14 +1880,18 @@ int main(int argc, char *argv[]) if (mpi_rank == 0) { print_usage(argv[0]); } - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 0; } if (strcmp(argv[i], "-V") == 0 || strcmp(argv[i], "--version") == 0) { if (mpi_rank == 0) { print_version(); } - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 0; } if (strcmp(argv[i], "--usage") == 0) { @@ -1891,7 +1981,9 @@ int main(int argc, char *argv[]) free(usage_flag); free(dummy_sep); free(dummy_app); - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 0; } @@ -2037,7 +2129,9 @@ int main(int argc, char *argv[]) fclose(output_fp); } - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 0; } @@ -2120,22 +2214,49 @@ int main(int argc, char *argv[]) // Parse and analyze mount points (all ranks need this for SSH test) std::vector mounts = parse_mounts(); - // Collect header warnings (will add $TMPDIR and SSH checks later) + // Collect header warnings std::vector header_warnings; + // Generate PYTHONHOME warning/note (only rank 0 needs to do this since it's the only one printing) + if (mpi_rank == 0) { + const char* pythonhome_env = getenv("PYTHONHOME"); + if (!pythonhome_env || pythonhome_env[0] == '\0') { + header_warnings.push_back("WARNING: The PYTHONHOME environment variable was not set."); + } else { + // Check if paths are valid + std::vector py_paths = discover_python_prefixes(); + if (py_paths.empty()) { + header_warnings.push_back("WARNING: None of the PYTHONHOME paths are valid."); + } else { + header_warnings.push_back("Note: At least one path from PYTHONHOME is valid and contains Python packages."); + } + } + } + + // Generate Munge warning/note (only rank 0) + if (mpi_rank == 0) { + const char* sandbox_engine = getenv("SANDBOX_ENGINE"); + bool in_sandbox = (sandbox_engine != nullptr); + + if (check_munge_available(in_sandbox)) { + header_warnings.push_back("Note: Munge is working as expected."); + } else { + header_warnings.push_back("WARNING: Munge is either not set up or not supported."); + } + } + // ALL RANKS: Test SSH connectivity using ring topology (uses collective MPI operations) - if (mpi_size > 1) { + // MPI/SSH warning will be generated by test_ssh_ring or afterward + if (!is_mpi_job) { + // Not running under MPI + if (mpi_rank == 0) { + header_warnings.push_back("WARNING: Not executing under MPI. This environment may not match what is used by parallel programs."); + } + } else if (mpi_size > 1) { test_ssh_ring(mpi_rank, mpi_size, header_warnings); } else if (mpi_rank == 0) { - // Single node - test local SSH only - std::string hostname = get_hostname_for_ssh(); - std::string ssh_error; - if (!test_ssh_connectivity(hostname, ssh_error)) { - header_warnings.push_back("SSH test failed for " + hostname + ": " + ssh_error); - header_warnings.push_back(" (SSH keys may not be configured for compute node access)"); - } else { - header_warnings.push_back("SSH test passed for " + hostname); - } + // Single node - add single node warning + header_warnings.push_back("WARNING: Executing under MPI with a single compute node. This may not give accurate ssh results."); } // Only rank 0 generates and prints output @@ -2150,7 +2271,9 @@ int main(int argc, char *argv[]) if (stat(output_file.c_str(), &st) == 0 && !overwrite) { std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 1; } @@ -2159,7 +2282,9 @@ int main(int argc, char *argv[]) if (!output_fp) { std::cerr << "Error: Could not open output file '" << output_file << "': " << strerror(errno) << std::endl; - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 1; } @@ -2184,6 +2309,8 @@ int main(int argc, char *argv[]) } } - MPI_Finalize(); + if (is_mpi_job) { + MPI_Finalize(); + } return 0; } From d68c45ee703d04d7ddd959996dafe0c7a1dfbe2a Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:17:57 -0800 Subject: [PATCH 12/18] Phase 6, small output tweaks. --- src/config_helper/spindle-config-helper.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 685fbb69..beeb407e 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -939,9 +939,20 @@ static void test_ssh_ring(int mpi_rank, int mpi_size, std::vector& } } + // Count unique nodes from the hostname list + std::set unique_nodes; + for (int i = 0; i < mpi_size; i++) { + std::string hostname(&all_hosts[i * MAX_HOSTNAME_LEN]); + if (!hostname.empty()) { + unique_nodes.insert(hostname); + } + } + int num_nodes = unique_nodes.size(); + if (all_passed) { char msg[512]; - snprintf(msg, sizeof(msg), "Note: Tested ssh successfully under MPI using %d nodes.", mpi_size); + snprintf(msg, sizeof(msg), "Note: Tested ssh successfully under MPI using %d ranks across %d nodes.", + mpi_size, num_nodes); warnings.push_back(std::string(msg)); } else { warnings.push_back("WARNING: Executing under MPI, but ssh communication among compute nodes failed."); @@ -1378,9 +1389,9 @@ static void generate_default_config(const ConfigMap& confmap, output_path_option(confCommPaths, "commpaths"); output_path_option(confCachePaths, "cachepaths"); - // Add comment block for pythonpath + // Add comment block for python-prefix std::cout << "#" << std::endl; - std::cout << "# pythonpath" << std::endl; + std::cout << "# python-prefix" << std::endl; std::cout << "# This is a colon-separated list of paths that contain python packages (including site-specific" << std::endl; std::cout << "# packages). The directories will include PYTHONHOME (if specified) and the results of" << std::endl; std::cout << "# checking python's sys.prefix and sys.path. If the entry is commented out, the program could" << std::endl; From dac40aef2aa64908a942380665f536d27e44c47a Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:26:28 -0800 Subject: [PATCH 13/18] Phase 6 chunk 1 --- src/config_helper/spindle-config-helper.cc | 664 +-------------------- 1 file changed, 1 insertion(+), 663 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index beeb407e..b53eb0e5 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -401,105 +401,6 @@ static PathAnalysis analyze_path(const std::string& path, const std::vector& mounts, - const ConfigMap& confmap) -{ - std::cout << "# Spindle filesystem recommendations:" << std::endl; - - // Show local writable mounts - std::vector local_mounts; - for (const auto& mp : mounts) { - if (!mp.is_remote) { - local_mounts.push_back(&mp); - } - } - - if (!local_mounts.empty()) { - std::cout << "# Local: "; - bool first = true; - for (const auto* mp : local_mounts) { - if (!first) std::cout << ", "; - std::cout << mp->path << " (" << mp->fstype << ", " - << mp->avail_gb << "GB)"; - first = false; - } - std::cout << std::endl; - } else { - std::cout << "# WARNING: No suitable local filesystems found!" << std::endl; - } - - std::cout << "#" << std::endl; - - // Analyze each configured option - auto analyze_option = [&](SpindleConfigID config_id, const char* name, bool check_python = false) { - if (!confmap.isSet(config_id)) { - return; - } - - auto result = confmap.getValueString(config_id); - if (!result.first || result.second.empty()) { - return; - } - - std::string paths = result.second; - std::vector analyses; - - // Split on colons for multi-path options - size_t pos = 0; - while (pos < paths.length()) { - size_t end = paths.find(':', pos); - if (end == std::string::npos) { - end = paths.length(); - } - - std::string path = paths.substr(pos, end - pos); - if (!path.empty()) { - analyses.push_back(analyze_path(path, mounts, check_python)); - } - - pos = end + 1; - } - - if (analyses.empty()) { - return; - } - - std::cout << "# " << name << " analysis:" << std::endl; - - for (const auto& analysis : analyses) { - const char* status = analysis.is_ok ? "OK" : "WARNING"; - std::cout << "# " << status << ": " << analysis.original; - - if (!analysis.resolved.empty()) { - std::cout << " -> " << analysis.resolved; - if (analysis.mount && !analysis.mount->is_remote) { - std::cout << " (" << analysis.mount->fstype << ", " - << analysis.mount->avail_gb << "GB)"; - } else if (analysis.mount && analysis.mount->is_remote) { - std::cout << " (" << analysis.mount->fstype << ")"; - } - } else if (!analysis.expanded.empty()) { - std::cout << " -> " << analysis.expanded; - } else { - std::cout << " -> (empty)"; - } - - std::cout << std::endl; - - for (const auto& warning : analysis.warnings) { - std::cout << "# " << warning << std::endl; - } - } - - std::cout << "#" << std::endl; - }; - - analyze_option(confCommPaths, "commpaths"); - analyze_option(confCachePaths, "cachepaths"); - analyze_option(confPythonPrefix, "pythonprefix", true); - analyze_option(confLocalPrefix, "localprefix"); -} - static std::string get_timestamp() { time_t now = time(nullptr); @@ -616,39 +517,6 @@ static std::string determine_best_path(const std::vector& mounts, return ""; } -static std::string get_python_prefix_from_which(const char* python_cmd) -{ - // Run 'which python' or 'which python3' - std::string cmd = std::string("which ") + python_cmd + " 2>/dev/null"; - FILE* fp = popen(cmd.c_str(), "r"); - if (!fp) { - return ""; - } - - char path[PATH_MAX]; - std::string result; - if (fgets(path, sizeof(path), fp) != nullptr) { - // Remove trailing newline - size_t len = strlen(path); - if (len > 0 && path[len-1] == '\n') { - path[len-1] = '\0'; - } - - // Resolve symlinks - char resolved[PATH_MAX]; - if (realpath(path, resolved) != nullptr) { - // Get parent directory (strip /bin/python*) - std::string full_path = resolved; - size_t bin_pos = full_path.rfind("/bin/"); - if (bin_pos != std::string::npos) { - result = full_path.substr(0, bin_pos); - } - } - } - pclose(fp); - return result; -} - // Get Python paths from sys.prefix and sys.path static std::vector get_python_sys_paths() { @@ -1029,141 +897,6 @@ static void print_config_header(std::vector& warnings); // Parse a spindle.conf file and return key-value pairs // Returns: map of key -> value pairs // Ignores comments and blank lines -static std::map parse_config_file(const std::string& filepath, - std::vector& warnings) -{ - std::map config; - std::ifstream file(filepath); - - if (!file.is_open()) { - warnings.push_back("WARNING: Could not open config file: " + filepath); - return config; - } - - std::string line; - int line_num = 0; - while (std::getline(file, line)) { - line_num++; - - // Trim leading/trailing whitespace - size_t start = line.find_first_not_of(" \t\r\n"); - if (start == std::string::npos) continue; // Empty line - - size_t end = line.find_last_not_of(" \t\r\n"); - line = line.substr(start, end - start + 1); - - // Skip comments and empty lines - if (line.empty() || line[0] == '#') continue; - - // Find '=' separator - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) { - warnings.push_back("WARNING: Malformed line " + std::to_string(line_num) + - " in " + filepath + ": " + line); - continue; - } - - // Extract key and value - std::string key = line.substr(0, eq_pos); - std::string value = line.substr(eq_pos + 1); - - // Trim whitespace around key and value - start = key.find_first_not_of(" \t"); - end = key.find_last_not_of(" \t"); - if (start != std::string::npos) { - key = key.substr(start, end - start + 1); - } - - start = value.find_first_not_of(" \t"); - end = value.find_last_not_of(" \t"); - if (start != std::string::npos) { - value = value.substr(start, end - start + 1); - } - - if (!key.empty()) { - config[key] = value; - } - } - - file.close(); - return config; -} - -// Parse command-line options from argv (options after --) -// Returns: map of key -> value pairs -static std::map parse_cmdline_options(int argc, char* argv[], - int separator_idx, - std::vector& warnings) -{ - std::map options; - - if (separator_idx == -1 || separator_idx + 1 >= argc) { - return options; // No options after -- - } - - // Parse options after -- - for (int i = separator_idx + 1; i < argc; i++) { - std::string arg = argv[i]; - - // Skip --usage - if (arg == "--usage") continue; - - // Parse --key=value or --key value - if (arg.substr(0, 2) == "--") { - std::string key, value; - size_t eq_pos = arg.find('='); - - if (eq_pos != std::string::npos) { - // --key=value - key = arg.substr(2, eq_pos - 2); - value = arg.substr(eq_pos + 1); - } else { - // --key (boolean or next arg is value) - key = arg.substr(2); - - // Check if next arg exists and doesn't start with -- - if (i + 1 < argc && argv[i + 1][0] != '-') { - value = argv[i + 1]; - i++; // Skip next arg - } else { - // Boolean flag - value = "true"; - } - } - - // Remove dashes from key to match config file format - key.erase(std::remove(key.begin(), key.end(), '-'), key.end()); - - if (!key.empty()) { - options[key] = value; - } - } else { - warnings.push_back("WARNING: Ignoring unrecognized argument: " + arg); - } - } - - return options; -} - -// Output a checked/validated config (for check mode) -static void output_checked_config(const std::map& config, - std::vector& warnings) -{ - // Print header - print_config_header(warnings); - - // Output all options in sorted order - std::vector keys; - for (const auto& kv : config) { - keys.push_back(kv.first); - } - std::sort(keys.begin(), keys.end()); - - for (const auto& key : keys) { - std::cout << key << " = " << config.at(key) << std::endl; - } -} - static void print_config_header(std::vector& warnings) { std::cout << "# spindle.conf generated by spindle-config-helper version " @@ -1461,119 +1194,6 @@ static void generate_default_config(const ConfigMap& confmap, } } -static void generate_all_config(const ConfigMap& confmap, - const std::vector& mounts, - std::vector& header_warnings) -{ - // Print header - print_config_header(header_warnings); - - // Get all Spindle options - const std::map& all_opts = getOptionsByID(); - - // Iterate through all options and output them - for (const auto& pair : all_opts) { - SpindleConfigID id = pair.first; - const SpindleOption& opt = pair.second; - - // Skip options without command-line names (internal options) - if (!opt.cmdline_long) continue; - - // Get the config file key name (cmdline_long with dashes removed) - std::string key = opt.cmdline_long; - key.erase(std::remove(key.begin(), key.end(), '-'), key.end()); - - // Check if this option is set (either by user or default) - bool is_set = confmap.isSet(id); - std::string value; - - if (is_set) { - // Get the actual value - auto result = confmap.getValueString(id); - if (result.first) { - value = result.second; - } - } - - // For the 4 path variables, output with full analysis - if (id == confCommPaths || id == confCachePaths || - id == confPythonPrefix || id == confLocalPrefix) { - // Use the existing detailed output from generate_default_config - continue; // We'll call those separately - } - - // For other options, output as comment if not explicitly set by user - // or uncommented if set by command line - std::string comment_prefix = is_set ? "" : "# "; - - // Output the option - std::cout << comment_prefix << key << " = " << value << std::endl; - } - - // Now output the 4 path variables with full analysis - // These are always uncommented and analyzed - auto output_path_option = [&](SpindleConfigID config_id, const char* name, - bool check_python = false) { - if (!confmap.isSet(config_id)) { - return; - } - - auto result = confmap.getValueString(config_id); - if (!result.first || result.second.empty()) { - return; - } - - std::string paths = result.second; - std::vector comments; - - // For pythonprefix, discover additional Python installations - std::vector discovered_pythons; - if (config_id == confPythonPrefix) { - discovered_pythons = discover_python_prefixes(); - } - - // Analyze paths (simplified for all mode) - std::vector path_list; - size_t pos = 0; - while (pos < paths.length()) { - size_t end = paths.find(':', pos); - if (end == std::string::npos) { - end = paths.length(); - } - std::string path = paths.substr(pos, end - pos); - if (!path.empty()) { - path_list.push_back(path); - } - pos = end + 1; - } - - // Add discovered Python paths - if (config_id == confPythonPrefix) { - for (const auto& pypath : discovered_pythons) { - if (std::find(path_list.begin(), path_list.end(), pypath) == path_list.end()) { - path_list.push_back(pypath); - } - } - } - - // Output - std::string output_value; - for (size_t i = 0; i < path_list.size(); i++) { - if (i > 0) output_value += ":"; - output_value += path_list[i]; - } - - if (!output_value.empty()) { - print_path_variable(name, output_value, comments); - } - }; - - output_path_option(confCommPaths, "commpaths"); - output_path_option(confCachePaths, "cachepaths"); - output_path_option(confPythonPrefix, "pythonprefix", true); - output_path_option(confLocalPrefix, "localprefix"); -} - static void print_version() { std::cout << "spindle-config-helper version " << PACKAGE_VERSION << std::endl; @@ -1611,83 +1231,6 @@ static bool check_munge_available(bool in_sandbox) return false; // fork failed } -static void print_security_analysis(const ConfigMap& confmap, const std::vector& mounts) -{ - std::cout << "# Security configuration:" << std::endl; - - // Detect if we're in a sandbox - const char* sandbox_engine = getenv("SANDBOX_ENGINE"); - bool in_sandbox = (sandbox_engine != NULL); - - // Get the current security setting - auto sec_result = confmap.getValueString(confSecurity); - std::string current_mode = "unknown"; - if (sec_result.first) { - current_mode = sec_result.second; - } - - std::cout << "# Current mode: " << current_mode << std::endl; - - // Validate based on mode - if (current_mode == "none") { - std::cout << "# WARNING: No authentication enabled (connections are insecure)" << std::endl; - } else if (current_mode == "keyfile") { - const char* keydir = get_sec_keydir_path(); - if (keydir) { - std::cout << "# Keyfile directory: " << keydir << std::endl; - - // Check if keydir is on a shared (remote) filesystem - std::string keydir_resolved = resolve_real_path(keydir); - const MountPoint* mp = find_mount_for_path(keydir_resolved, mounts); - - if (mp) { - if (mp->is_remote) { - std::cout << "# OK: Keyfile directory is on shared filesystem (" - << mp->fstype << ")" << std::endl; - } else { - std::cout << "# WARNING: Keyfile directory is on LOCAL filesystem (" - << mp->fstype << ")" << std::endl; - std::cout << "# Keyfile must be on a filesystem shared across all nodes" << std::endl; - } - } else { - std::cout << "# WARNING: Cannot determine filesystem type for keyfile directory" << std::endl; - } - } else { - std::cout << "# Note: Keyfile mode not compiled in" << std::endl; - } - } else if (current_mode == "launchmon") { - std::cout << "# WARNING: LaunchMON security mode (deprecated/untested)" << std::endl; - } else if (current_mode == "munge") { - std::cout << "# Munge authentication enabled" << std::endl; - } - - std::cout << "#" << std::endl; - std::cout << "# Munge availability:" << std::endl; - - // Always check for munge as a recommendation - bool munge_binary_exists = (access("/usr/bin/munge", X_OK) == 0); - - if (munge_binary_exists) { - if (in_sandbox) { - std::cout << "# Munge binary found at /usr/bin/munge" << std::endl; - std::cout << "# Note: Cannot test munge daemon (running in sandbox)" << std::endl; - } else { - bool munge_works = check_munge_available(false); - if (munge_works) { - std::cout << "# Munge is available and working (recommended security option)" << std::endl; - } else { - std::cout << "# Munge binary found but daemon is not running" << std::endl; - std::cout << "# Munge provides secure authentication (recommended if available)" << std::endl; - } - } - } else { - std::cout << "# Munge not found" << std::endl; - std::cout << "# Munge provides secure authentication (recommended if available)" << std::endl; - } - - std::cout << "#" << std::endl; -} - static void print_usage(const char *progname) { std::cout << "Usage: " << progname << " [OPTIONS] [-- SPINDLE_OPTIONS]\n\n" @@ -1772,59 +1315,6 @@ static void print_usage(const char *progname) << std::endl; } -static void format_config_value(const ConfigMap &confmap, SpindleConfigID id, - const SpindleOption &opt) -{ - if (!confmap.isSet(id)) { - return; - } - - std::string value; - bool has_value = false; - - switch (opt.conftype) { - case cvBool: { - auto result = confmap.getValueBool(id); - if (result.first) { - value = result.second ? "yes" : "no"; - has_value = true; - } - break; - } - case cvInteger: { - auto result = confmap.getValueIntegral(id); - if (result.first) { - value = std::to_string(result.second); - has_value = true; - } - break; - } - case cvString: - case cvStringOptional: - case cvList: - case cvEnum: { - auto result = confmap.getValueString(id); - if (result.first) { - value = result.second; - has_value = true; - } - break; - } - } - - if (has_value && opt.cmdline_long) { - // Convert cmdline option name to config file format - // e.g., "comm-paths" -> "commpaths" - std::string key = opt.cmdline_long; - // Remove dashes using erase-remove idiom - key.erase(std::remove_if(key.begin(), key.end(), - [](char c) { return c == '-'; }), - key.end()); - - std::cout << key << " = " << value << std::endl; - } -} - // Detect if we're running under an MPI launcher by checking for MPI-related environment variables static bool running_under_mpi() { @@ -1998,154 +1488,6 @@ int main(int argc, char *argv[]) return 0; } - // Handle check mode with separate code path - if (check_mode) { - std::map config_values; - std::vector warnings; - - // Parse config file if provided - if (!check_file.empty()) { - config_values = parse_config_file(check_file, warnings); - } - - // Parse command-line options after -- - std::map cmdline_options = - parse_cmdline_options(argc, argv, separator_idx, warnings); - - // Merge: command-line overrides file - for (const auto& kv : cmdline_options) { - config_values[kv.first] = kv.second; - } - - // Validate all options by building argv and using gatherAllConfigInfo - // This reuses Spindle's validation without needing getOptionsByID - - std::vector val_argv; - std::vector val_strings; // Keep strings alive - - val_argv.push_back(argv[0]); // Program name - - // Build command-line arguments for validation - // Map config file keys (no dashes) to command-line format (with dashes) - // Use a static mapping to avoid iterating through option lists - std::map key_mapping = { - {"localprefix", "local-prefix"}, - {"pythonprefix", "python-prefix"}, - // Note: commpaths and cachepaths don't need mapping - they're single words - {"relocaout", "reloc-aout"}, - {"reloclibs", "reloc-libs"}, - {"relocpython", "reloc-python"}, - {"relocexec", "reloc-exec"}, - {"followfork", "follow-fork"}, - {"stopreloc", "stop-reloc"}, - {"pushpull", "push"}, - {"shmcachesize", "shmcache-size"}, - {"msgcachebuffer", "msgcache-buffer"}, - {"msgcachetimeout", "msgcache-timeout"}, - {"cleanupproc", "cleanup-proc"}, - {"cacheprefix", "cache-prefix"}, - {"hostbin", "host-bin"}, - {"startmultisession", "start-multisession"}, - {"startsession", "start-session"}, - {"endsession", "end-session"}, - {"runinsession", "run-in-session"}, - {"patchldso", "patch-ldso"}, - {"crashdedup", "crash-dedup"}, - {"crashaltstack", "crash-altstack"}, - {"crashlog", "crash-log"}, - {"rshcommand", "rsh-command"}, - {"enablersh", "enable-rsh"}, - {"execexcludes", "exec-excludes"}, - {"numaincludes", "numa-includes"}, - {"numaexcludes", "numa-excludes"}, - {"audittype", "audit-type"}, - {"disablelogging", "disable-logging"}, - {"noclean", "no-clean"}, - {"nohide", "no-hide"} - // Add more as needed - most single-word keys don't need mapping - }; - - for (const auto& kv : config_values) { - std::string cmdline_key = kv.first; - - // Check if we have a mapping for this key - auto it = key_mapping.find(kv.first); - if (it != key_mapping.end()) { - cmdline_key = it->second; - } - - // Add as two separate args: --key value - std::string opt = "--" + cmdline_key; - val_strings.push_back(opt); - val_strings.push_back(kv.second); - } - - for (auto& s : val_strings) { - val_argv.push_back(const_cast(s.c_str())); - } - - // Add dummy app - char *val_sep = strdup("--"); - char *val_app = strdup("/bin/true"); - val_argv.push_back(val_sep); - val_argv.push_back(val_app); - - char **val_argv_ptr = val_argv.data(); - int val_argc = val_argv.size(); - - ConfigMap val_confmap; - std::string val_errmsg; - if (!gatherAllConfigInfo(val_argc, val_argv_ptr, false, val_confmap, val_errmsg)) { - // Validation failed - gatherAllConfigInfo already printed error to stderr - // Do not generate config output - free(val_sep); - free(val_app); - return 1; - } - - free(val_sep); - free(val_app); - - // Handle file output if specified - FILE* output_fp = nullptr; - int saved_stdout = -1; - - if (!output_file.empty()) { - struct stat st; - if (stat(output_file.c_str(), &st) == 0 && !overwrite) { - std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; - std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; - return 1; - } - - output_fp = fopen(output_file.c_str(), "w"); - if (!output_fp) { - std::cerr << "Error: Could not open output file '" << output_file - << "': " << strerror(errno) << std::endl; - return 1; - } - - saved_stdout = dup(STDOUT_FILENO); - dup2(fileno(output_fp), STDOUT_FILENO); - } - - // Output the validated config - output_checked_config(config_values, warnings); - - // Restore stdout and close file if we redirected - if (output_fp) { - fflush(stdout); - dup2(saved_stdout, STDOUT_FILENO); - close(saved_stdout); - fclose(output_fp); - } - - if (is_mpi_job) { - MPI_Finalize(); - } - return 0; - } - // Reconstruct argv to move Spindle options before -- and add dummy app // Input: spindle-config-helper [config-helper-opts] -- [spindle-opts] // Output: spindle-config-helper [spindle-opts] -- /bin/true (filtering out config-helper opts) @@ -2305,11 +1647,7 @@ int main(int argc, char *argv[]) } // Generate config based on mode - if (all_mode) { - generate_all_config(confmap, mounts, header_warnings); - } else { - generate_default_config(confmap, mounts, header_warnings, mpi_rank, mpi_size); - } + generate_default_config(confmap, mounts, header_warnings, mpi_rank, mpi_size); // Restore stdout and close file if we redirected if (output_fp) { From 716dd343caa362f326e367c854484ee18097f097 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:31:08 -0800 Subject: [PATCH 14/18] Phase 6 chunk 1b --- src/config_helper/spindle-config-helper.cc | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index b53eb0e5..dc802280 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -1364,8 +1364,6 @@ int main(int argc, char *argv[]) int separator_idx = -1; std::string output_file; bool overwrite = false; - std::string check_file; - bool check_mode = false; bool all_mode = false; // First pass: check for help/version and find separator, parse output options @@ -1415,25 +1413,6 @@ int main(int argc, char *argv[]) if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--all") == 0) { all_mode = true; } - if (strcmp(argv[i], "-c") == 0) { - check_mode = true; - // Check if next arg is a file (not starting with -) - if (i + 1 < argc && argv[i + 1][0] != '-') { - check_file = argv[i + 1]; - i++; // Skip next argument - } - } - if (strncmp(argv[i], "--check=", 8) == 0) { - check_mode = true; - check_file = argv[i] + 8; - } - if (strcmp(argv[i], "--check") == 0) { - check_mode = true; - if (i + 1 < argc && argv[i + 1][0] != '-') { - check_file = argv[i + 1]; - i++; // Skip next argument - } - } } else { // After --: check for --usage if (strcmp(argv[i], "--usage") == 0) { From a528e52055bfaa1e08d7f56cb3822ac000b78ef5 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:33:59 -0800 Subject: [PATCH 15/18] Phase 6 1c --- src/config_helper/spindle-config-helper.cc | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index dc802280..13883129 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -1364,7 +1364,6 @@ int main(int argc, char *argv[]) int separator_idx = -1; std::string output_file; bool overwrite = false; - bool all_mode = false; // First pass: check for help/version and find separator, parse output options for (int i = 1; i < argc; i++) { @@ -1410,9 +1409,6 @@ int main(int argc, char *argv[]) if (strcmp(argv[i], "--overwrite") == 0) { overwrite = true; } - if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--all") == 0) { - all_mode = true; - } } else { // After --: check for --usage if (strcmp(argv[i], "--usage") == 0) { @@ -1476,11 +1472,7 @@ int main(int argc, char *argv[]) auto is_config_helper_opt = [](const char* arg) { return strcmp(arg, "-o") == 0 || - strcmp(arg, "-c") == 0 || - strcmp(arg, "-a") == 0 || strncmp(arg, "--output", 8) == 0 || - strncmp(arg, "--check", 7) == 0 || - strcmp(arg, "--all") == 0 || strcmp(arg, "--overwrite") == 0; }; @@ -1489,8 +1481,7 @@ int main(int argc, char *argv[]) for (int i = 1; i < separator_idx; i++) { if (is_config_helper_opt(argv[i])) { // Skip this arg, and if it takes an argument, skip that too - if ((strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0 || - strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--check") == 0) && + if ((strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0) && i + 1 < argc && argv[i + 1][0] != '-') { i++; // Skip the file argument too } From 46df3ba4fd36c9e6bc09d33591f8b7544650894f Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:37:06 -0800 Subject: [PATCH 16/18] Phase 6 1d --- src/config_helper/spindle-config-helper.cc | 98 ++-------------------- 1 file changed, 7 insertions(+), 91 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 13883129..52645608 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -1362,10 +1362,7 @@ int main(int argc, char *argv[]) // Also check for -- --usage which should show Spindle's usage bool show_spindle_usage = false; int separator_idx = -1; - std::string output_file; - bool overwrite = false; - - // First pass: check for help/version and find separator, parse output options + // First pass: check for help/version and find separator for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "--") == 0) { separator_idx = i; @@ -1395,20 +1392,6 @@ int main(int argc, char *argv[]) if (strcmp(argv[i], "--usage") == 0) { show_spindle_usage = true; } - if (strcmp(argv[i], "-o") == 0 && i + 1 < argc) { - output_file = argv[i + 1]; - i++; // Skip next argument - } - if (strncmp(argv[i], "--output=", 9) == 0) { - output_file = argv[i] + 9; - } - if (strcmp(argv[i], "--output") == 0 && i + 1 < argc) { - output_file = argv[i + 1]; - i++; // Skip next argument - } - if (strcmp(argv[i], "--overwrite") == 0) { - overwrite = true; - } } else { // After --: check for --usage if (strcmp(argv[i], "--usage") == 0) { @@ -1470,44 +1453,17 @@ int main(int argc, char *argv[]) std::vector new_argv; new_argv.push_back(argv[0]); // Program name - auto is_config_helper_opt = [](const char* arg) { - return strcmp(arg, "-o") == 0 || - strncmp(arg, "--output", 8) == 0 || - strcmp(arg, "--overwrite") == 0; - }; + // No config-helper options to filter anymore (only -h/-V which exit early) + // Just copy all args, respecting the -- separator if (separator_idx != -1) { - // Add config-helper options (before --), filtering out our options - for (int i = 1; i < separator_idx; i++) { - if (is_config_helper_opt(argv[i])) { - // Skip this arg, and if it takes an argument, skip that too - if ((strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0) && - i + 1 < argc && argv[i + 1][0] != '-') { - i++; // Skip the file argument too - } - continue; - } - new_argv.push_back(argv[i]); - } - - // Add spindle options (after --) + // Add args before -- (should be nothing since -h/-V exit early) + // And add spindle options (after --) for (int i = separator_idx + 1; i < argc; i++) { new_argv.push_back(argv[i]); } - } else { - // No -- found, copy args but filter out config-helper options - for (int i = 1; i < argc; i++) { - if (is_config_helper_opt(argv[i])) { - if ((strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0 || - strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--check") == 0) && - i + 1 < argc && argv[i + 1][0] != '-') { - i++; // Skip the file argument too - } - continue; - } - new_argv.push_back(argv[i]); - } } + // If no --, nothing to add (no spindle options) // Add dummy app char *dummy_sep = strdup("--"); @@ -1584,48 +1540,8 @@ int main(int argc, char *argv[]) // Only rank 0 generates and prints output if (mpi_rank == 0) { - // Handle output file if specified - FILE* output_fp = nullptr; - int saved_stdout = -1; - - if (!output_file.empty()) { - // Check if file exists - struct stat st; - if (stat(output_file.c_str(), &st) == 0 && !overwrite) { - std::cerr << "Error: Output file '" << output_file << "' already exists." << std::endl; - std::cerr << "Use --overwrite to overwrite the existing file." << std::endl; - if (is_mpi_job) { - MPI_Finalize(); - } - return 1; - } - - // Open output file - output_fp = fopen(output_file.c_str(), "w"); - if (!output_fp) { - std::cerr << "Error: Could not open output file '" << output_file - << "': " << strerror(errno) << std::endl; - if (is_mpi_job) { - MPI_Finalize(); - } - return 1; - } - - // Redirect stdout to file - saved_stdout = dup(STDOUT_FILENO); - dup2(fileno(output_fp), STDOUT_FILENO); - } - - // Generate config based on mode + // Generate config to stdout generate_default_config(confmap, mounts, header_warnings, mpi_rank, mpi_size); - - // Restore stdout and close file if we redirected - if (output_fp) { - fflush(stdout); - dup2(saved_stdout, STDOUT_FILENO); - close(saved_stdout); - fclose(output_fp); - } } if (is_mpi_job) { From edadcd21c13627ad925ac81f36f2497efb2cbedb Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:44:37 -0800 Subject: [PATCH 17/18] Phase 6 1e --- src/config_helper/spindle-config-helper.cc | 81 +++------------------- 1 file changed, 10 insertions(+), 71 deletions(-) diff --git a/src/config_helper/spindle-config-helper.cc b/src/config_helper/spindle-config-helper.cc index 52645608..4156f60b 100644 --- a/src/config_helper/spindle-config-helper.cc +++ b/src/config_helper/spindle-config-helper.cc @@ -1235,79 +1235,18 @@ static void print_usage(const char *progname) { std::cout << "Usage: " << progname << " [OPTIONS] [-- SPINDLE_OPTIONS]\n\n" << "Generate and validate Spindle configuration files for HPC environments.\n\n" - << "DESCRIPTION:\n" - << " spindle-config-helper analyzes the local system and generates appropriate\n" - << " spindle.conf files. It detects local vs. remote filesystems, discovers\n" - << " Python installations, validates paths, and tests SSH connectivity.\n\n" + << "INSTRUCTIONS:\n" + << " 1) Set PYTHONPREFIX to the python your programs will use.\n" + << " 2) Set TMPDIR to node-local temporary storage (e.g., /tmp).\n" + << " 3) Run " << progname << " as you would any other parallel\n" + << " program, e.g., flux run -N2 -n2 " << progname << ".\n" + << " 4) Check the output, then and redirect/copypaste to\n" + << " ~/.spindle/spindle.conf\n\n" + << "See the output for explanations of how spindle-config-helper chooses\n" + << " paths and validates the environment.\n\n" << "OPTIONS:\n" << " -h, --help Display this help message and exit\n" - << " -V, --version Display version information and exit\n" - << " -o, --output=FILE Write output to FILE instead of stdout\n" - << " --overwrite Allow overwriting existing output file\n" - << " -c, --check[=FILE] Validate existing config FILE (optional)\n" - << " -a, --all Generate complete config with all options\n" - << " --usage Display Spindle's usage summary\n\n" - << "MODES:\n" - << " Default mode (no options):\n" - << " - Analyzes filesystem mounts (local vs. remote)\n" - << " - Checks $TMPDIR availability and recommends paths\n" - << " - Tests SSH connectivity to verify compute node access\n" - << " - Discovers Python installations via 'which python/python3'\n" - << " - Generates localprefix from top-level directories\n" - << " - Outputs minimal spindle.conf with 4 path variables:\n" - << " * commpaths: Local directories for inter-process communication\n" - << " * cachepaths: Local directories for file caching\n" - << " * pythonprefix: Python installation directories\n" - << " * localprefix: Directories excluded from caching\n" - << " - Includes warnings and recommendations as comments\n\n" - << " Check mode (-c [FILE]):\n" - << " - FILE is optional: validates and outputs options from:\n" - << " * Config file (if FILE provided)\n" - << " * Command-line options after --\n" - << " - Command-line options override config file values\n" - << " - Validates option names and values\n" - << " - Use cases:\n" - << " * -c FILE : Validate file\n" - << " * -c -- --opt=val : Validate single option\n" - << " * -c FILE -- --opt : Validate file + override\n\n" - << " All mode (-a):\n" - << " - Enumerates all Spindle configuration options\n" - << " - Comments out options with default values (prefix '#')\n" - << " - Uncomments options specified after --\n" - << " - Useful for discovering available options\n\n" - << "ENVIRONMENT:\n" - << " $TMPDIR Preferred directory for commpaths/cachepaths\n" - << " Falls back to /tmp if not set\n\n" - << "SPINDLE_OPTIONS:\n" - << " Any valid Spindle option can be passed after --\n" - << " Use '" << progname << " -- --usage' to see all available options\n" - << " Common options:\n" - << " --commpaths=PATHS Colon-separated list of comm directories\n" - << " --cachepaths=PATHS Colon-separated list of cache directories\n" - << " --level=LEVEL Aggressiveness: high, medium, low, off\n" - << " --security=MODE Security: none, munge, keyfile\n" - << " --launcher=TYPE Launcher: serial, slurm, flux, etc.\n\n" - << "EXAMPLES:\n" - << " # Generate new config to stdout\n" - << " " << progname << "\n\n" - << " # Generate config for review before saving\n" - << " " << progname << " | less\n\n" - << " # Save config to file\n" - << " " << progname << " -o ~/.spindle/spindle.conf\n\n" - << " # Generate with custom paths\n" - << " " << progname << " -- --commpaths=/dev/shm --cachepaths=/tmp\n\n" - << " # Check existing config for warnings\n" - << " " << progname << " -c ~/.spindle/spindle.conf | grep WARNING\n\n" - << " # Update config with new security setting\n" - << " " << progname << " -c ~/.spindle/spindle.conf -o ~/.spindle/spindle.conf \\\n" - << " --overwrite -- --security=munge\n\n" - << " # Generate complete config with all options\n" - << " " << progname << " -a -o spindle.conf.full\n\n" - << " # See all available Spindle options\n" - << " " << progname << " -- --usage\n\n" - << "EXIT STATUS:\n" - << " 0 Success\n" - << " 1 Error (invalid arguments, file I/O error, etc.)\n\n" + << " -V, --version Display version information and exit\n\n" << "SEE ALSO:\n" << " spindle(1), spindle-config-helper(1)\n\n" << "VERSION:\n" From 4ee5c7763b28a5cf28bfa0a42a7459ad287ef886 Mon Sep 17 00:00:00 2001 From: Barry Rountree Date: Fri, 6 Nov 2026 09:52:50 -0800 Subject: [PATCH 18/18] Phase 6 man page --- src/config_helper/man/spindle-config-helper.1 | 299 +++++++----------- 1 file changed, 108 insertions(+), 191 deletions(-) diff --git a/src/config_helper/man/spindle-config-helper.1 b/src/config_helper/man/spindle-config-helper.1 index 837e5a73..cb6446f4 100644 --- a/src/config_helper/man/spindle-config-helper.1 +++ b/src/config_helper/man/spindle-config-helper.1 @@ -1,12 +1,12 @@ -.TH SPINDLE-CONFIG-HELPER 1 "October 2026" "Spindle 0.13" "User Commands" +.TH SPINDLE-CONFIG-HELPER 1 "November 2026" "Spindle 0.13" "User Commands" .SH NAME spindle-config-helper \- generate and validate Spindle configuration files .SH SYNOPSIS .B spindle-config-helper -[\fIOPTIONS\fR] [\fB\-\-\fR \fISPINDLE_OPTIONS\fR] +[\fB-h\fR|\fB--help\fR] [\fB-V\fR|\fB--version\fR] [\fB\-\-\fR \fISPINDLE_OPTIONS\fR] .SH DESCRIPTION .B spindle-config-helper -is a utility for generating and validating Spindle configuration files in HPC +is a utility for generating Spindle configuration files in HPC environments. It analyzes the local system to automatically detect optimal configuration settings and outputs a .I spindle.conf @@ -14,181 +14,116 @@ file suitable for the environment. .PP The tool performs the following analysis: .IP \(bu 2 -Detects local vs. remote filesystem mounts (NFS, Lustre, GPFS, tmpfs) +Detects local vs. remote filesystem mounts (NFS, Lustre, GPFS, tmpfs, overlay) .IP \(bu 2 Validates .B $TMPDIR availability and recommends appropriate paths .IP \(bu 2 -Tests SSH connectivity to verify compute node access +Tests SSH connectivity between MPI ranks to verify compute node access .IP \(bu 2 -Discovers Python installations via system PATH +Discovers Python installations via +.B $PYTHONPREFIX +and Python's +.B sys.prefix +and +.B sys.path .IP \(bu 2 Generates .I localprefix -from top-level directories without remote mounts +from top-level local directories .IP \(bu 2 Validates path writability and accessibility +.IP \(bu 2 +Tests Munge availability and functionality .PP -The tool should be run on compute nodes in the same environment where -Spindle will execute, as login/front-end nodes may have different characteristics. +The tool must be run as an MPI program on at least two compute nodes +to provide accurate environment analysis, particularly for SSH connectivity testing. .SH OPTIONS .TP .BR \-h ", " \-\-help -Display detailed usage information and exit. +Display usage information and exit. .TP .BR \-V ", " \-\-version Display version information and exit. -.TP -.BR \-o ", " \-\-output =\fIFILE\fR -Write output to -.I FILE -instead of stdout. By default, fails if -.I FILE -already exists. -.TP -.BR \-\-overwrite -Allow overwriting an existing output file when used with -.BR \-o . -.TP -.BR \-c ", " \-\-check [=\fIFILE\fR] -Validate and merge Spindle configuration options. -.I FILE -is optional. Reads options from config file (if provided) and/or -command-line (after -.BR \-\- ). -Command-line options override config file values. Validates option names -and values, outputting the merged and validated configuration. -.TP -.BR \-a ", " \-\-all -Generate a complete configuration file including all Spindle options. -Options with default values are commented out (prefixed with '#'). -Options specified after -.B \-\- -are uncommented. Useful for discovering available options. -.TP -.BR \-\-usage -Display Spindle's usage summary with all available configuration options. -.SH MODES -.SS Default Mode -Without any mode flags, +.SH INSTRUCTIONS +To generate a Spindle configuration file: +.IP 1. 4 +Set +.B PYTHONPREFIX +to the Python installation your programs will use. +.IP 2. 4 +Set +.B TMPDIR +to node-local temporary storage (e.g., +.IR /tmp ). +.IP 3. 4 +Run .B spindle-config-helper -generates a minimal configuration with four essential path variables: +as you would any other parallel program, e.g.: +.RS 4 +.nf +flux run -N2 -n2 spindle-config-helper +srun -N2 -n2 spindle-config-helper +.fi +.RE +.IP 4. 4 +Review the output, then redirect or copy/paste to +.IR ~/.spindle/spindle.conf . +.PP +See the tool's output for explanations of how it chooses paths and validates the environment. +.SH CONFIGURATION OUTPUT +The tool generates a minimal configuration with four essential path variables: .TP .B commpaths -Local directories for inter-process communication. Prefers +Local directories for inter-process communication. Based on .B $TMPDIR -if set, otherwise recommends a suitable local filesystem like -.I /tmp -or -.IR /dev/shm . +if set and valid, otherwise recommends a suitable local filesystem. .TP .B cachepaths Local directories for file caching. Uses similar logic to .BR commpaths . .TP -.B pythonprefix +.B python-prefix Python installation directories discovered via -.B which python +.B $PYTHONPREFIX +and Python's +.B sys.prefix and -.BR "which python3" . -Validates that -.I bin/python -or -.I bin/python3 -exists under each directory. +.BR sys.path . +Validates that paths contain Python packages. .TP .B localprefix Colon-separated list of top-level directories that should be excluded from -caching. Generated from -.I /proc -.I /sys -.I /dev -and other top-level directories without remote mounts underneath. +caching. Generated from local (non-remote) filesystem mount points. +Overlay filesystems are treated as non-local. .PP -The output includes warnings and recommendations as comments. -.SS Check Mode -When invoked with -.BR \-c , -the tool validates and merges configuration options. The filename is optional, -allowing three use cases: -.TP -.B \-c FILE -Reads and validates options from -.IR FILE . -.TP -.B \-c \-\- \-\-option=value -Validates individual command-line options without reading a file. -Useful for testing whether an option exists and if its value is valid. -.TP -.B \-c FILE \-\- \-\-option=value -Reads options from -.IR FILE , -merges with command-line options (command-line wins), validates all options, -and outputs the merged configuration. -.PP -The tool validates: -.IP \(bu 2 -Option names are valid in the current Spindle version +The output includes warnings and notes as comments: .IP \(bu 2 -Enum values (e.g., level must be high/medium/low/off/default) +.B WARNING: +Issues that require attention (missing environment variables, failed tests, invalid paths) .IP \(bu 2 -Reports malformed lines, unknown options, and invalid values -.SS All Mode -When invoked with -.BR \-a , -the tool enumerates all Spindle configuration options (~49 options). -Options with default values are commented out, while options specified -on the command line after -.B \-\- -are uncommented with their specified values. +.B Note: +Successful validations and test results .SH ENVIRONMENT .TP +.B PYTHONPREFIX +Python installation prefix. Preferred source for Python path discovery. +.TP .B TMPDIR Preferred directory for .B commpaths and .BR cachepaths . -If not set, the tool recommends -.I /tmp -or another suitable local filesystem. -.TP -.B SANDBOX_ENGINE -If set, indicates the tool is running in a sandbox environment. -Affects capability testing (e.g., Munge daemon connectivity cannot be -verified in a sandbox). +If not set or invalid, the tool will issue a warning and attempt to find +suitable alternatives. .SH SPINDLE OPTIONS Any valid Spindle configuration option can be passed after .BR \-\- . +These options are parsed using Spindle's built-in option handling. Use .B spindle-config-helper \-\- \-\-usage -to see all available options. Common options include: -.TP -.BI \-\-commpaths= PATHS -Colon-separated list of directories for inter-process communication. -Should be on local (not remote) filesystems. -.TP -.BI \-\-cachepaths= PATHS -Colon-separated list of directories for file caching. -Should be on local filesystems with sufficient space. -.TP -.BI \-\-level= LEVEL -Aggressiveness of library redistribution: -.BR high ", " medium ", " low ", " off ", or " default . -.TP -.BI \-\-security= MODE -Security mode: -.BR none ", " munge ", " keyfile ", or " launchmon . -.TP -.BI \-\-launcher= TYPE -Resource manager: -.BR serial ", " slurm ", " flux ", " lsf ", " alps ", " openmpi ", " wreckrun ", " jsrun ", " hostbin ", or " srun . -.TP -.BI \-\-python\-prefix= PATHS -Colon-separated list of Python installation prefixes. -.TP -.BI \-\-local\-prefix= PATHS -Colon-separated list of directory prefixes to exclude from caching. +to see all available options. .SH EXAMPLES .TP Generate a new configuration to stdout: @@ -198,54 +133,25 @@ spindle-config-helper .RE .fi .TP -Generate and save to file: -.nf -.RS -spindle-config-helper -o ~/.spindle/spindle.conf -.RE -.fi -.TP -Generate with custom paths: -.nf -.RS -spindle-config-helper -- --commpaths=/dev/shm --cachepaths=/tmp -.RE -.fi -.TP -Check existing config for warnings: +Generate configuration under MPI: .nf .RS -spindle-config-helper -c ~/.spindle/spindle.conf | grep WARNING +flux run -N2 -n2 spindle-config-helper > ~/.spindle/spindle.conf +srun -N2 -n2 spindle-config-helper | tee spindle.conf .RE .fi .TP -Update existing config with new security setting: +Check warnings in output: .nf .RS -spindle-config-helper -c ~/.spindle/spindle.conf \\ - -o ~/.spindle/spindle.conf --overwrite \\ - -- --security=munge -.RE -.fi -.TP -Generate complete config with all options: -.nf -.RS -spindle-config-helper -a -o spindle.conf.full -.RE -.fi -.TP -Generate config with high aggressiveness: -.nf -.RS -spindle-config-helper -a -- --level=high --launcher=slurm +spindle-config-helper | grep WARNING .RE .fi .TP View all available Spindle options: .nf .RS -spindle-config-helper -- --usage +spindle-config-helper \-\- \-\-usage .RE .fi .SH FILES @@ -254,49 +160,60 @@ spindle-config-helper -- --usage System-wide Spindle configuration file. .TP .I ~/.spindle/spindle.conf -Per-user Spindle configuration file (common location). +Per-user Spindle configuration file (recommended location). .TP .I /proc/mounts Used to detect filesystem mount points and types. .SH EXIT STATUS .TP .B 0 -Success. Valid configuration was generated or validated. +Success. Valid configuration was generated. .TP .B 1 -Error. Invalid arguments, file I/O error, or configuration parse error occurred. +Error. Invalid arguments or configuration error occurred. .SH NOTES +.SS MPI Requirement +The tool must be run as an MPI program on at least two nodes for accurate results. +Running on a single node will issue a warning, as SSH ring testing requires +multiple nodes to properly validate inter-node connectivity. .SS Path Validation The tool validates paths by checking: .IP \(bu 2 -Parent directory is writable +Filesystem is local (not NFS, Lustre, GPFS, or overlay) .IP \(bu 2 -Path is on a local (not remote) filesystem +Path is writable by the current user .IP \(bu 2 -Path can be created and deleted (test file creation) +Parent directories exist .PP Paths that fail validation are filtered out and reported in warnings. -.SS SSH Testing -The tool tests SSH connectivity to the local hostname to verify that -compute node access is properly configured. This test uses -.BR "ssh -o BatchMode=yes" -to check for key-based authentication without password prompts. +.SS SSH Ring Testing +When running under MPI with multiple nodes, each rank tests SSH connectivity +to the next rank in a ring pattern (highest rank connects to rank 0). +This verifies that compute nodes can communicate via SSH without password prompts. .SS Python Discovery -Python installations are discovered by running -.B which python +Python installations are discovered in order of preference: +.IP 1. 4 +.B $PYTHONPREFIX +environment variable +.IP 2. 4 +Python's +.B sys.prefix +(queried from Python itself) +.IP 3. 4 +Python's +.B sys.path +entries +.PP +The tool validates that discovered paths contain actual Python packages, +not just the Python binary. +.SS Overlay Filesystems +Overlay filesystems (detected via +.IR /proc/mounts ) +are treated as non-local and excluded from +.BR commpaths , +.BR cachepaths , and -.BR "which python3" , -following symlinks to find the actual installation directories. -The tool validates that -.I bin/python -or -.I bin/python3 -exists under each discovered prefix. -.SS Sandbox Environments -When running in a sandbox environment (detected via -.BR SANDBOX_ENGINE ), -some tests are limited. For example, Munge daemon connectivity cannot -be verified, though Munge binary availability is still checked. +.BR localprefix . .SH SEE ALSO .BR spindle (1) .PP