feat(gateway): merge drop-in config fragments over the base file

TESTING.md

@@ -580,25 +580,24 @@ An environment overrides gateway settings by writing the tables to change to
 `/etc/openshell-test/gateway-overrides.toml` in a playbook that runs before the
 installer. The file holds whole tables, such as `[openshell.gateway]` or
 `[openshell.drivers.podman]`, and is not a complete configuration. Each
-installer applies it to the gateway's own file:
+installer installs it as the drop-in fragment `gateway.d/90-test-overrides.toml`
+next to the gateway's own `gateway.toml`, and the gateway merges it over that
+file. The base file is never edited:
 
-- `binaries` applies it to the `/etc/openshell/gateway.toml` it generates.
-- `deb` and `rpm` apply it to the service user's
+- `binaries` installs it beside the `/etc/openshell/gateway.toml` it generates.
+- `deb` and `rpm` install it beside the service user's
   `~/.config/openshell/gateway.toml`, the file the packaged service reads. The
-  file starts as a copy of the rpm's
-  `/usr/share/openshell-gateway/gateway.toml.default`, where the service would
-  seed it on first start, or as a minimal `version = 2` file when the package
-  has no default. No environment replaces the packaged file, and an environment
-  with no overrides leaves it untouched.
+  rpm still seeds that file from
+  `/usr/share/openshell-gateway/gateway.toml.default` on first start. A package
+  with no default gets a minimal `version = 2` base file, because a fragment
+  doesn't declare a version. An environment with no overrides leaves the
+  packaged configuration untouched.
 - `k3s` installers ignore the file.
 
-`tests/tmachine/ansible/roles/openshell_gateway/files/merge-gateway-toml.py`
-does the work. Each key in the overrides replaces the same key in the table, or
-is added to it, and a table the file lacks is appended. The rest of the file,
-comments included, stays. The script parses the result and fails the setup,
-leaving the file unchanged, when it isn't valid TOML. It can't remove a key.
-Settings that the packaged service reads from `gateway.env` instead of the
-TOML are not covered.
+A key in the fragment replaces the same key in the base file's table, or is
+added to it, and a table the base lacks is added. Arrays replace the base
+array. A fragment can't remove a key. Settings that the packaged service reads
+from `gateway.env` instead of the TOML are not covered.
 
 The `oci-image` feature suite (`tests/suites/features/oci-image`) checks OCI
 image identity and working-directory behavior shared by the Docker and Podman

crates/openshell-server/src/cli.rs

@@ -65,6 +65,10 @@ struct ConfigPreflightArgs {
     #[arg(long, conflicts_with = "gateway_args")]
     path: Option<PathBuf>,
 
+    /// Explicit drop-in directory. Defaults to `gateway.d` next to the base file.
+    #[arg(long, conflicts_with = "gateway_args")]
+    config_dir: Option<PathBuf>,
+
     /// Gateway daemon arguments to replay after `--`.
     #[arg(last = true, allow_hyphen_values = true, value_name = "GATEWAY_ARGS")]
     gateway_args: Vec<OsString>,
@@ -81,6 +85,14 @@ struct RunArgs {
     #[arg(long, env = "OPENSHELL_GATEWAY_CONFIG")]
     config: Option<PathBuf>,
 
+    /// Directory of `*.toml` drop-in fragments merged over the base file.
+    ///
+    /// Fragments apply in lexical order and a later fragment wins. Tables
+    /// merge key by key; scalars and arrays replace. Defaults to `gateway.d`
+    /// next to the base file when that directory exists.
+    #[arg(long, env = "OPENSHELL_GATEWAY_CONFIG_DIR")]
+    config_dir: Option<PathBuf>,
+
     /// Operator-assigned name for this gateway installation.
     #[arg(
         long = "name",
@@ -423,11 +435,15 @@ fn prepare_server_config_with_drivers(
     // Load TOML when explicitly requested, or from the default XDG location
     // when that file exists. Missing default config is not an error: runtime
     // defaults and OPENSHELL_* env vars are enough for package-managed starts.
-    let config_path = resolve_config_path(args)?;
-    let mut file: Option<ConfigFile> = if let Some(path) = config_path {
-        Some(config_file::load(&path).map_err(|e| miette::miette!("{e}"))?)
-    } else {
+    let sources = resolve_config_sources(args, None, None)?;
+    let mut file: Option<ConfigFile> = if sources.is_empty() {
         None
+    } else {
+        Some(
+            config_file::load_layered(sources.base.as_deref(), sources.dir.as_deref())
+                .map_err(|e| miette::miette!("{e}"))?
+                .file,
+        )
     };
     if let Some(file) = file.as_ref() {
         merge_file_into_args(args, &file.openshell.gateway, matches);
@@ -1041,7 +1057,13 @@ fn run_config_preflight_with_drivers(
     compute_drivers: &ComputeDriverRegistry,
 ) -> Result<Option<crate::ConfiguredComputeDriver>> {
     if args.gateway_args.is_empty() {
-        return run_effective_config_preflight(args.path, run, matches, compute_drivers);
+        return run_effective_config_preflight(
+            args.path,
+            args.config_dir,
+            run,
+            matches,
+            compute_drivers,
+        );
     }
 
     let replay_matches = match command().try_get_matches_from(
@@ -1065,24 +1087,30 @@ fn run_config_preflight_with_drivers(
         // configuration. Let the immediately following invocation perform it.
         return Ok(None);
     }
-    run_effective_config_preflight(None, replay.run, &replay_matches, compute_drivers)
+    run_effective_config_preflight(None, None, replay.run, &replay_matches, compute_drivers)
 }
 
 fn run_effective_config_preflight(
     path_override: Option<PathBuf>,
+    dir_override: Option<PathBuf>,
     mut run: RunArgs,
     matches: &ArgMatches,
     compute_drivers: &ComputeDriverRegistry,
 ) -> Result<Option<crate::ConfiguredComputeDriver>> {
-    let path = if path_override.is_some() {
-        path_override
+    let sources = resolve_config_sources(&run, path_override, dir_override)?;
+    // Name the base file (or the directory) in the stable failure message.
+    let path = sources.base.clone().or_else(|| sources.dir.clone());
+    let mut file = if sources.is_empty() {
+        None
     } else {
-        resolve_config_path(&run)?
+        let layered =
+            config_file::preflight_layered(sources.base.as_deref(), sources.dir.as_deref())
+                .map_err(|error| miette::miette!("{error}"))?;
+        for source in &layered.sources {
+            println!("config: merged {}", source.display());
+        }
+        Some(layered.file)
     };
-    let mut file = path
-        .as_ref()
-        .map(|path| config_file::preflight(path).map_err(|error| miette::miette!("{error}")))
-        .transpose()?;
     if let Some(file) = file.as_ref() {
         merge_file_into_args(&mut run, &file.openshell.gateway, matches);
     }
@@ -1386,21 +1414,54 @@ fn validate_preflight_semantics(
     Ok(())
 }
 
-fn resolve_config_path(args: &RunArgs) -> Result<Option<PathBuf>> {
-    if let Some(path) = args.config.clone() {
-        return Ok(Some(path));
-    }
+/// Where gateway configuration is read from: an optional base file and an
+/// optional directory of `*.toml` drop-in fragments merged over it.
+#[derive(Debug, Default)]
+struct ConfigSources {
+    base: Option<PathBuf>,
+    dir: Option<PathBuf>,
+}
 
-    let default_path = defaults::default_gateway_config_path()?;
-    match std::fs::symlink_metadata(&default_path) {
-        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
-        // Fail closed when the path exists or cannot be inspected. Returning it
-        // lets the shared loader produce the same safe, path-specific error used
-        // for an explicit configuration instead of treating it as absent.
-        Ok(_) | Err(_) => Ok(Some(default_path)),
+impl ConfigSources {
+    fn is_empty(&self) -> bool {
+        self.base.is_none() && self.dir.is_none()
     }
 }
 
+fn resolve_config_sources(
+    args: &RunArgs,
+    base_override: Option<PathBuf>,
+    dir_override: Option<PathBuf>,
+) -> Result<ConfigSources> {
+    let base = if let Some(path) = base_override.or_else(|| args.config.clone()) {
+        Some(path)
+    } else {
+        let default_path = defaults::default_gateway_config_path()?;
+        match std::fs::symlink_metadata(&default_path) {
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
+            // Fail closed when the path exists or cannot be inspected.
+            // Returning it lets the shared loader produce the same safe,
+            // path-specific error used for an explicit configuration.
+            Ok(_) | Err(_) => Some(default_path),
+        }
+    };
+    // An explicit directory must exist. Otherwise use `gateway.d` beside the
+    // base file (or beside the default location) when it is there.
+    let dir = if let Some(dir) = dir_override.or_else(|| args.config_dir.clone()) {
+        Some(dir)
+    } else {
+        let anchor = match base.clone() {
+            Some(path) => path,
+            None => defaults::default_gateway_config_path()?,
+        };
+        anchor
+            .parent()
+            .map(|parent| parent.join("gateway.d"))
+            .filter(|dir| dir.is_dir())
+    };
+    Ok(ConfigSources { base, dir })
+}
+
 fn apply_runtime_defaults(args: &mut RunArgs) -> Result<Option<LocalTlsPaths>> {
     let local_tls = if args.disable_tls {
         None
@@ -3633,13 +3694,23 @@ require_client_auth = true
                 let tmp = std::path::PathBuf::from(std::env::var_os("XDG_CONFIG_HOME").unwrap());
 
                 let (args, _) = parse_with_args(&["openshell-gateway"]);
-                assert_eq!(super::resolve_config_path(&args).unwrap(), None);
+                assert_eq!(
+                    super::resolve_config_sources(&args, None, None)
+                        .unwrap()
+                        .base,
+                    None
+                );
 
                 let config = tmp.join("openshell").join("gateway.toml");
                 std::fs::create_dir_all(config.parent().unwrap()).unwrap();
                 std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap();
 
-                assert_eq!(super::resolve_config_path(&args).unwrap(), Some(config));
+                assert_eq!(
+                    super::resolve_config_sources(&args, None, None)
+                        .unwrap()
+                        .base,
+                    Some(config)
+                );
             });
     }
 
@@ -3652,12 +3723,106 @@ require_client_auth = true
                     parse_with_args(&["openshell-gateway", "--config", "/tmp/missing.toml"]);
 
                 assert_eq!(
-                    super::resolve_config_path(&args).unwrap(),
+                    super::resolve_config_sources(&args, None, None)
+                        .unwrap()
+                        .base,
                     Some(std::path::PathBuf::from("/tmp/missing.toml"))
                 );
             });
     }
 
+    #[test]
+    fn config_dir_defaults_to_gateway_d_beside_the_base_file() {
+        let tmp = tempfile::tempdir().unwrap();
+        let base = tmp.path().join("gateway.toml");
+        std::fs::write(&base, "[openshell]\nversion = 2\n").unwrap();
+        Environment::new()
+            .remove("OPENSHELL_GATEWAY_CONFIG")
+            .remove("OPENSHELL_GATEWAY_CONFIG_DIR")
+            .run(|| {
+                let (args, _) =
+                    parse_with_args(&["openshell-gateway", "--config", base.to_str().unwrap()]);
+                assert_eq!(
+                    super::resolve_config_sources(&args, None, None)
+                        .unwrap()
+                        .dir,
+                    None,
+                    "a missing gateway.d is not an error"
+                );
+                std::fs::create_dir(tmp.path().join("gateway.d")).unwrap();
+                assert_eq!(
+                    super::resolve_config_sources(&args, None, None)
+                        .unwrap()
+                        .dir,
+                    Some(tmp.path().join("gateway.d"))
+                );
+            });
+    }
+
+    #[test]
+    fn explicit_config_dir_overrides_the_implicit_directory() {
+        let tmp = tempfile::tempdir().unwrap();
+        let other = tmp.path().join("other");
+        Environment::new()
+            .remove("OPENSHELL_GATEWAY_CONFIG")
+            .remove("OPENSHELL_GATEWAY_CONFIG_DIR")
+            .run(|| {
+                let (args, _) = parse_with_args(&[
+                    "openshell-gateway",
+                    "--config-dir",
+                    other.to_str().unwrap(),
+                ]);
+                assert_eq!(
+                    super::resolve_config_sources(&args, None, None)
+                        .unwrap()
+                        .dir,
+                    Some(other.clone())
+                );
+            });
+    }
+
+    #[test]
+    fn preflight_merges_fragments_and_names_a_bad_fragment() {
+        let tmp = tempfile::tempdir().unwrap();
+        let base = tmp.path().join("gateway.toml");
+        std::fs::write(&base, "[openshell]\nversion = 2\n").unwrap();
+        let dir = tmp.path().join("gateway.d");
+        std::fs::create_dir(&dir).unwrap();
+        std::fs::write(
+            dir.join("10-ok.toml"),
+            "[openshell.gateway]\nlog_level = 'debug'\n",
+        )
+        .unwrap();
+        let xdg = tempfile::tempdir().unwrap();
+        Environment::new()
+            .set("XDG_CONFIG_HOME", xdg.path().to_str().unwrap())
+            .set("XDG_STATE_HOME", xdg.path().to_str().unwrap())
+            .remove("OPENSHELL_GATEWAY_CONFIG")
+            .remove("OPENSHELL_GATEWAY_CONFIG_DIR")
+            .remove("OPENSHELL_COMPUTE_DRIVER")
+            .remove("OPENSHELL_DRIVERS")
+            .run(|| {
+                let args = || super::ConfigPreflightArgs {
+                    path: Some(base.clone()),
+                    ..Default::default()
+                };
+                let (run, matches) = parse_with_args(&["openshell-gateway", "--disable-tls"]);
+                super::run_config_preflight(args(), run, &matches).unwrap();
+
+                std::fs::write(
+                    dir.join("20-bad.toml"),
+                    "[openshell.gateway]\nnot_a_field = 1\n",
+                )
+                .unwrap();
+                let (run, matches) = parse_with_args(&["openshell-gateway", "--disable-tls"]);
+                let error = super::run_config_preflight(args(), run, &matches)
+                    .unwrap_err()
+                    .to_string();
+                assert!(error.contains("category=malformed"), "{error}");
+                assert!(error.contains("20-bad.toml"), "{error}");
+            });
+    }
+
     #[test]
     fn runtime_defaults_populate_database_url_from_xdg_state() {
         let tmp = tempfile::tempdir().unwrap();
@@ -3864,8 +4029,14 @@ require_client_auth = true
             )
             .unwrap();
         let (preflight_args, preflight_matches) = parse_with_args(&argv);
-        super::run_effective_config_preflight(None, preflight_args, &preflight_matches, &registry)
-            .expect("selected or candidate Podman preflight accepts the legacy alias");
+        super::run_effective_config_preflight(
+            None,
+            None,
+            preflight_args,
+            &preflight_matches,
+            &registry,
+        )
+        .expect("selected or candidate Podman preflight accepts the legacy alias");
         assert!(!state.path().join("network-supervisor").exists());
         let prepared = super::prepare_server_config_with_drivers(&mut run, &matches, &registry)
             .expect("global network trust should normalize during startup preparation");
@@ -3946,7 +4117,7 @@ require_client_auth = true
                         argv.extend(["--compute-driver", "docker"]);
                     }
                     let (run, matches) = parse_with_args(&argv);
-                    super::run_effective_config_preflight(None, run, &matches, &registry)
+                    super::run_effective_config_preflight(None, None, run, &matches, &registry)
                         .expect("inactive Podman settings must not affect preflight");
                     let (mut run, matches) = parse_with_args(&argv);
                     let prepared =