From fc5e1190f06b7c1f25ec5fe55703bc2979388b83 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 20 Jan 2026 16:33:05 -0500 Subject: [PATCH 01/22] Started merging service binaries --- bin/Cargo.toml | 8 ++------ bin/cli.rs | 2 +- bin/{pbs.rs => commit-boost.rs} | 18 ++++++++++++++--- crates/cli/src/docker_init.rs | 16 ++++++++------- crates/cli/src/lib.rs | 12 ++++------- crates/common/src/config/constants.rs | 10 ++------- crates/common/src/config/mod.rs | 2 +- crates/common/src/config/pbs.rs | 29 +++++++++++++++++++++------ crates/common/src/config/signer.rs | 23 ++++----------------- tests/src/utils.rs | 7 +++---- 10 files changed, 64 insertions(+), 63 deletions(-) rename bin/{pbs.rs => commit-boost.rs} (83%) diff --git a/bin/Cargo.toml b/bin/Cargo.toml index d50c2932..0871c67b 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -23,9 +23,5 @@ name = "commit-boost-cli" path = "cli.rs" [[bin]] -name = "commit-boost-pbs" -path = "pbs.rs" - -[[bin]] -name = "commit-boost-signer" -path = "signer.rs" +name = "commit-boost" +path = "commit-boost.rs" diff --git a/bin/cli.rs b/bin/cli.rs index 234dc9bd..0f36d388 100644 --- a/bin/cli.rs +++ b/bin/cli.rs @@ -18,7 +18,7 @@ async fn main() -> eyre::Result<()> { color_eyre::install()?; // set default backtrace unless provided - let args = cb_cli::Args::parse(); + let args = cb_cli::CliArgs::parse(); args.run().await } diff --git a/bin/pbs.rs b/bin/commit-boost.rs similarity index 83% rename from bin/pbs.rs rename to bin/commit-boost.rs index 0b7c3f72..a217a404 100644 --- a/bin/pbs.rs +++ b/bin/commit-boost.rs @@ -3,7 +3,7 @@ use cb_common::{ utils::{initialize_tracing_log, wait_for_signal}, }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; -use clap::Parser; +use clap::{Parser, Subcommand}; use eyre::Result; use tracing::{error, info}; @@ -13,7 +13,19 @@ const VERSION: &str = concat!("v", env!("CARGO_PKG_VERSION")); /// Subcommands and global arguments for the module #[derive(Parser, Debug)] #[command(name = "Commit-Boost PBS Service", version = VERSION, about, long_about = None)] -struct Cli {} +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Run the PBS service + Pbs, + + /// Run the Signer service + Signer, +} #[tokio::main] async fn main() -> Result<()> { @@ -25,7 +37,7 @@ async fn main() -> Result<()> { let _guard = initialize_tracing_log(PBS_MODULE_NAME, LogsSettings::from_env_config()?); - let _args = cb_cli::PbsArgs::parse(); + let _args = cb_cli::CbArgs::parse(); let pbs_config = load_pbs_config().await?; diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index b535e54d..811a581e 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -6,15 +6,15 @@ use std::{ use cb_common::{ config::{ - CHAIN_SPEC_ENV, CONFIG_DEFAULT, CONFIG_ENV, CommitBoostConfig, DIRK_CA_CERT_DEFAULT, - DIRK_CA_CERT_ENV, DIRK_CERT_DEFAULT, DIRK_CERT_ENV, DIRK_DIR_SECRETS_DEFAULT, - DIRK_DIR_SECRETS_ENV, DIRK_KEY_DEFAULT, DIRK_KEY_ENV, JWTS_ENV, LOGS_DIR_DEFAULT, - LOGS_DIR_ENV, LogsSettings, METRICS_PORT_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, ModuleKind, - PBS_ENDPOINT_ENV, PBS_MODULE_NAME, PROXY_DIR_DEFAULT, PROXY_DIR_ENV, + CB_MODULE_NAME, CHAIN_SPEC_ENV, CONFIG_DEFAULT, CONFIG_ENV, CommitBoostConfig, + DIRK_CA_CERT_DEFAULT, DIRK_CA_CERT_ENV, DIRK_CERT_DEFAULT, DIRK_CERT_ENV, + DIRK_DIR_SECRETS_DEFAULT, DIRK_DIR_SECRETS_ENV, DIRK_KEY_DEFAULT, DIRK_KEY_ENV, JWTS_ENV, + LOGS_DIR_DEFAULT, LOGS_DIR_ENV, LogsSettings, METRICS_PORT_ENV, MODULE_ID_ENV, + MODULE_JWT_ENV, ModuleKind, PBS_ENDPOINT_ENV, PROXY_DIR_DEFAULT, PROXY_DIR_ENV, PROXY_DIR_KEYS_DEFAULT, PROXY_DIR_KEYS_ENV, PROXY_DIR_SECRETS_DEFAULT, PROXY_DIR_SECRETS_ENV, SIGNER_DEFAULT, SIGNER_DIR_KEYS_DEFAULT, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS_DEFAULT, SIGNER_DIR_SECRETS_ENV, SIGNER_ENDPOINT_ENV, SIGNER_KEYS_ENV, - SIGNER_MODULE_NAME, SIGNER_PORT_DEFAULT, SIGNER_URL_ENV, SignerConfig, SignerType, + SIGNER_PORT_DEFAULT, SIGNER_URL_ENV, SignerConfig, SignerType, }, pbs::{BUILDER_V1_API_PATH, GET_STATUS_PATH}, signer::{ProxyStore, SignerLoader}, @@ -233,7 +233,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re // volumes pbs_volumes.extend(chain_spec_volume.clone()); - pbs_volumes.extend(get_log_volume(&cb_config.logs, PBS_MODULE_NAME)); + pbs_volumes.extend(get_log_volume(&cb_config.logs, CB_MODULE_NAME)); let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), @@ -577,6 +577,8 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re Ok(()) } +fn create_pbs_service() {} + /// FOO=${FOO} fn get_env_same(k: &str) -> (String, Option) { get_env_interp(k, k) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 34170470..f14ab11f 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -7,7 +7,7 @@ mod docker_init; #[derive(Parser, Debug)] #[command(version, about, long_about = LONG_ABOUT, name = "commit-boost-cli")] -pub struct Args { +pub struct CliArgs { #[command(subcommand)] pub cmd: Command, } @@ -26,7 +26,7 @@ pub enum Command { }, } -impl Args { +impl CliArgs { pub async fn run(self) -> eyre::Result<()> { print_logo(); @@ -41,9 +41,5 @@ impl Args { const LONG_ABOUT: &str = "Commit-Boost allows Ethereum validators to safely run MEV-Boost and community-built commitment protocols"; #[derive(Parser, Debug)] -#[command(version, about, long_about = LONG_ABOUT, name = "commit-boost-pbs")] -pub struct PbsArgs; - -#[derive(Parser, Debug)] -#[command(version, about, long_about = LONG_ABOUT, name = "commit-boost-signer")] -pub struct SignerArgs; +#[command(version, about, long_about = LONG_ABOUT, name = "commit-boost")] +pub struct CbArgs; diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index 67248596..bfbed8c1 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -14,20 +14,14 @@ pub const METRICS_PORT_ENV: &str = "CB_METRICS_PORT"; pub const LOGS_DIR_ENV: &str = "CB_LOGS_DIR"; pub const LOGS_DIR_DEFAULT: &str = "/var/logs/commit-boost"; -///////////////////////// PBS ///////////////////////// - -pub const PBS_IMAGE_DEFAULT: &str = "ghcr.io/commit-boost/pbs:latest"; -pub const PBS_MODULE_NAME: &str = "pbs"; +pub const CB_IMAGE_DEFAULT: &str = "ghcr.io/commit-boost/commit-boost:latest"; /// Where to receive BuilderAPI calls from beacon node pub const PBS_ENDPOINT_ENV: &str = "CB_PBS_ENDPOINT"; pub const MUX_PATH_ENV: &str = "CB_MUX_PATH"; -///////////////////////// SIGNER ///////////////////////// - -pub const SIGNER_IMAGE_DEFAULT: &str = "ghcr.io/commit-boost/signer:latest"; -pub const SIGNER_MODULE_NAME: &str = "signer"; +pub const CB_MODULE_NAME: &str = "commit-boost"; /// Where the signer module should open the server pub const SIGNER_ENDPOINT_ENV: &str = "CB_SIGNER_ENDPOINT"; diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 67a13cb0..3a3c23d8 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -41,7 +41,7 @@ pub struct CommitBoostConfig { impl CommitBoostConfig { /// Validate config pub async fn validate(&self) -> Result<()> { - self.pbs.pbs_config.validate(self.chain).await?; + self.pbs.validate(self.chain).await?; if let Some(signer) = &self.signer { signer.validate().await?; } diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 7bcf91e3..0e1848e5 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -10,18 +10,19 @@ use alloy::{ primitives::{U256, utils::format_ether}, providers::{Provider, ProviderBuilder}, }; +use docker_image::DockerImage; use eyre::{Result, ensure}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; use super::{ CommitBoostConfig, HTTP_TIMEOUT_SECONDS_DEFAULT, PBS_ENDPOINT_ENV, RuntimeMuxConfig, - constants::PBS_IMAGE_DEFAULT, load_optional_env_var, + constants::CB_IMAGE_DEFAULT, load_optional_env_var, }; use crate::{ commit::client::SignerClient, config::{ - CONFIG_ENV, MODULE_JWT_ENV, MuxKeysLoader, PBS_MODULE_NAME, PbsMuxes, SIGNER_URL_ENV, + CB_MODULE_NAME, CONFIG_ENV, MODULE_JWT_ENV, MuxKeysLoader, PbsMuxes, SIGNER_URL_ENV, load_env_var, load_file_from_env, }, pbs::{ @@ -204,7 +205,7 @@ impl PbsConfig { #[derive(Debug, Deserialize, Serialize)] pub struct StaticPbsConfig { /// Docker image of the module - #[serde(default = "default_pbs")] + #[serde(default = "default_image")] pub docker_image: String, /// Config of pbs module #[serde(flatten)] @@ -214,6 +215,22 @@ pub struct StaticPbsConfig { pub with_signer: bool, } +impl StaticPbsConfig { + /// Validate the static pbs config + pub async fn validate(&self, chain: Chain) -> Result<()> { + self.pbs_config.validate(chain).await?; + + // The Docker tag must parse + ensure!(!self.docker_image.is_empty(), "Docker image is empty"); + ensure!( + DockerImage::parse(&self.docker_image).is_ok(), + format!("Invalid Docker image: {}", self.docker_image) + ); + + Ok(()) + } +} + /// Runtime config for the pbs module #[derive(Debug, Clone)] pub struct PbsModuleConfig { @@ -237,8 +254,8 @@ pub struct PbsModuleConfig { pub mux_lookup: Option>, } -fn default_pbs() -> String { - PBS_IMAGE_DEFAULT.to_string() +fn default_image() -> String { + CB_IMAGE_DEFAULT.to_string() } /// Loads the default pbs config, i.e. with no signer client or custom data @@ -381,7 +398,7 @@ pub async fn load_pbs_custom_config() -> Result<(PbsModuleC Some(SignerClient::new( signer_server_url, module_jwt, - ModuleId(PBS_MODULE_NAME.to_string()), + ModuleId(CB_MODULE_NAME.to_string()), )?) } else { None diff --git a/crates/common/src/config/signer.rs b/crates/common/src/config/signer.rs index 8ab8186f..c5f70ac0 100644 --- a/crates/common/src/config/signer.rs +++ b/crates/common/src/config/signer.rs @@ -4,17 +4,16 @@ use std::{ path::PathBuf, }; -use docker_image::DockerImage; use eyre::{OptionExt, Result, bail, ensure}; use serde::{Deserialize, Serialize}; use tonic::transport::{Certificate, Identity}; use url::Url; use super::{ - CommitBoostConfig, SIGNER_ENDPOINT_ENV, SIGNER_IMAGE_DEFAULT, - SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_LIMIT_ENV, - SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_ENV, - SIGNER_PORT_DEFAULT, load_jwt_secrets, load_optional_env_var, utils::load_env_var, + CommitBoostConfig, SIGNER_ENDPOINT_ENV, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, + SIGNER_JWT_AUTH_FAIL_LIMIT_ENV, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, + SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_ENV, SIGNER_PORT_DEFAULT, load_jwt_secrets, + load_optional_env_var, utils::load_env_var, }; use crate::{ config::{DIRK_CA_CERT_ENV, DIRK_CERT_ENV, DIRK_DIR_SECRETS_ENV, DIRK_KEY_ENV}, @@ -32,9 +31,6 @@ pub struct SignerConfig { /// Port to listen for signer API calls on #[serde(default = "default_u16::")] pub port: u16, - /// Docker image of the module - #[serde(default = "default_signer_image")] - pub docker_image: String, /// Number of JWT auth failures before rate limiting an endpoint /// If set to 0, no rate limiting will be applied @@ -57,21 +53,10 @@ impl SignerConfig { // Port must be positive ensure!(self.port > 0, "Port must be positive"); - // The Docker tag must parse - ensure!(!self.docker_image.is_empty(), "Docker image is empty"); - ensure!( - DockerImage::parse(&self.docker_image).is_ok(), - format!("Invalid Docker image: {}", self.docker_image) - ); - Ok(()) } } -fn default_signer_image() -> String { - SIGNER_IMAGE_DEFAULT.to_string() -} - #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "snake_case")] pub struct DirkHostConfig { diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 58ef42cf..826ad769 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -7,9 +7,9 @@ use std::{ use alloy::primitives::U256; use cb_common::{ config::{ - PbsConfig, PbsModuleConfig, RelayConfig, SIGNER_IMAGE_DEFAULT, - SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, - SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, + PbsConfig, PbsModuleConfig, RelayConfig, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, + SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, + SignerType, StartSignerConfig, }, pbs::{RelayClient, RelayEntry}, signer::SignerLoader, @@ -109,7 +109,6 @@ pub fn get_signer_config(loader: SignerLoader) -> SignerConfig { SignerConfig { host: default_host(), port: SIGNER_PORT_DEFAULT, - docker_image: SIGNER_IMAGE_DEFAULT.to_string(), jwt_auth_fail_limit: SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, jwt_auth_fail_timeout_seconds: SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, inner: SignerType::Local { loader, store: None }, From 5036b7cbbc96fb5c0c4cddfc794f5eb8cd516a0b Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 27 Jan 2026 13:26:37 -0500 Subject: [PATCH 02/22] Started refactoring handle_docker_init() --- bin/commit-boost.rs | 4 +- bin/src/lib.rs | 2 +- crates/cli/src/docker_init.rs | 803 +++++++++++++++----------- crates/common/src/config/constants.rs | 2 + crates/common/src/config/mod.rs | 19 +- crates/common/src/config/pbs.rs | 19 +- examples/status_api/src/main.rs | 2 +- 7 files changed, 487 insertions(+), 364 deletions(-) diff --git a/bin/commit-boost.rs b/bin/commit-boost.rs index a217a404..3078a6f9 100644 --- a/bin/commit-boost.rs +++ b/bin/commit-boost.rs @@ -1,5 +1,5 @@ use cb_common::{ - config::{LogsSettings, PBS_MODULE_NAME, load_pbs_config}, + config::{LogsSettings, PBS_SERVICE_NAME, load_pbs_config}, utils::{initialize_tracing_log, wait_for_signal}, }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; @@ -35,7 +35,7 @@ async fn main() -> Result<()> { color_eyre::install()?; - let _guard = initialize_tracing_log(PBS_MODULE_NAME, LogsSettings::from_env_config()?); + let _guard = initialize_tracing_log(PBS_SERVICE_NAME, LogsSettings::from_env_config()?); let _args = cb_cli::CbArgs::parse(); diff --git a/bin/src/lib.rs b/bin/src/lib.rs index dd0f52c2..4bb668e6 100644 --- a/bin/src/lib.rs +++ b/bin/src/lib.rs @@ -6,7 +6,7 @@ pub mod prelude { SignedProxyDelegationBls, SignedProxyDelegationEcdsa, }, config::{ - LogsSettings, PBS_MODULE_NAME, StartCommitModuleConfig, load_builder_module_config, + LogsSettings, PBS_SERVICE_NAME, StartCommitModuleConfig, load_builder_module_config, load_commit_module_config, load_pbs_config, load_pbs_custom_config, }, signer::EcdsaSignature, diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 811a581e..a65e6740 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -10,11 +10,11 @@ use cb_common::{ DIRK_CA_CERT_DEFAULT, DIRK_CA_CERT_ENV, DIRK_CERT_DEFAULT, DIRK_CERT_ENV, DIRK_DIR_SECRETS_DEFAULT, DIRK_DIR_SECRETS_ENV, DIRK_KEY_DEFAULT, DIRK_KEY_ENV, JWTS_ENV, LOGS_DIR_DEFAULT, LOGS_DIR_ENV, LogsSettings, METRICS_PORT_ENV, MODULE_ID_ENV, - MODULE_JWT_ENV, ModuleKind, PBS_ENDPOINT_ENV, PROXY_DIR_DEFAULT, PROXY_DIR_ENV, - PROXY_DIR_KEYS_DEFAULT, PROXY_DIR_KEYS_ENV, PROXY_DIR_SECRETS_DEFAULT, + MODULE_JWT_ENV, ModuleKind, PBS_ENDPOINT_ENV, PBS_SERVICE_NAME, PROXY_DIR_DEFAULT, + PROXY_DIR_ENV, PROXY_DIR_KEYS_DEFAULT, PROXY_DIR_KEYS_ENV, PROXY_DIR_SECRETS_DEFAULT, PROXY_DIR_SECRETS_ENV, SIGNER_DEFAULT, SIGNER_DIR_KEYS_DEFAULT, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS_DEFAULT, SIGNER_DIR_SECRETS_ENV, SIGNER_ENDPOINT_ENV, SIGNER_KEYS_ENV, - SIGNER_PORT_DEFAULT, SIGNER_URL_ENV, SignerConfig, SignerType, + SIGNER_PORT_DEFAULT, SIGNER_SERVICE_NAME, SIGNER_URL_ENV, SignerConfig, SignerType, }, pbs::{BUILDER_V1_API_PATH, GET_STATUS_PATH}, signer::{ProxyStore, SignerLoader}, @@ -36,56 +36,120 @@ pub(super) const CB_ENV_FILE: &str = ".cb.env"; const SIGNER_NETWORK: &str = "signer_network"; +// Info about a custom chain spec to use +struct ServiceChainSpecInfo { + // Environment variable to set for the chain spec file's path + env: (String, Option), + + // Volume for binding the chain spec file into a container + volume: Volumes, +} + +// Info about the Commit-Boost config being used to create services +struct CommitBoostConfigInfo { + // Path to the config file + config_path: PathBuf, + + // Commit-Boost config + cb_config: CommitBoostConfig, + + // Volume for binding the config file into a container + config_volume: Volumes, +} + +// Information needed to create a Commit-Boost service +struct ServiceCreationInfo { + // Info about the Commit-Boost config being used + config_info: CommitBoostConfigInfo, + + // Environment variables to write in .env file + envs: IndexMap, + + // Targets to pass to prometheus + targets: Vec, + + // Warnings that need to be shown to the user + warnings: Vec, + + // JWTs for any modules owned by this service (TODO: are we going to offload modules to the + // user instead of owning them?) + jwts: IndexMap, + + // Custom chain spec info, if any + chain_spec: Option, + + // Next available port for metrics (TODO: this should be a setting in PBS and in Signer instead + // of a universal one) + metrics_port: u16, +} + +// Information about the created CB Signer service +struct SignerService { + // The created signer, as a docker compose service + service: Service, + + // The port used for metrics + metrics_port: u16, + + // The port used for signer API communication + signer_port: u16, +} + /// Builds the docker compose file for the Commit-Boost services // TODO: do more validation for paths, images, etc pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Result<()> { + let mut services = IndexMap::new(); + println!("Initializing Commit-Boost with config file: {}", config_path.display()); - let cb_config = CommitBoostConfig::from_file(&config_path)?; + let mut service_config = ServiceCreationInfo { + config_info: CommitBoostConfigInfo { + config_path: config_path.clone(), + cb_config: CommitBoostConfig::from_file(&config_path)?, + config_volume: Volumes::Simple(format!( + "./{}:{}:ro", + config_path.display(), + CONFIG_DEFAULT + )), + }, + envs: IndexMap::new(), + targets: Vec::new(), + warnings: Vec::new(), + jwts: IndexMap::new(), + chain_spec: None, + metrics_port: 9100, + }; + + // Validate the CB config + let cb_config = &mut service_config.config_info.cb_config; cb_config.validate().await?; + // Get the custom chain spec, if any let chain_spec_path = CommitBoostConfig::chain_spec_file(&config_path); - - let log_to_file = cb_config.logs.file.enabled; - let mut metrics_port = cb_config.metrics.as_ref().map(|m| m.start_port).unwrap_or_default(); - - let mut services = IndexMap::new(); - - // config volume to pass to all services - let config_volume = - Volumes::Simple(format!("./{}:{}:ro", config_path.display(), CONFIG_DEFAULT)); - let chain_spec_volume = chain_spec_path.as_ref().and_then(|p| { - // this is ok since the config has already been loaded once - let file_name = p.file_name()?.to_str()?; - Some(Volumes::Simple(format!("{}:/{}:ro", p.display(), file_name))) - }); - - let chain_spec_env = chain_spec_path.and_then(|p| { + if let Some(spec) = chain_spec_path { // this is ok since the config has already been loaded once - let file_name = p.file_name()?.to_str()?; - Some(get_env_val(CHAIN_SPEC_ENV, &format!("/{file_name}"))) - }); - - let mut jwts = IndexMap::new(); - // envs to write in .env file - let mut envs = IndexMap::new(); - // targets to pass to prometheus - let mut targets = Vec::new(); - - // address for signer API communication - let signer_port = cb_config.signer.as_ref().map(|s| s.port).unwrap_or(SIGNER_PORT_DEFAULT); - let signer_server = - if let Some(SignerConfig { inner: SignerType::Remote { url }, .. }) = &cb_config.signer { - url.to_string() - } else { - format!("http://cb_signer:{signer_port}") + let filename = spec.file_name().unwrap().to_str().unwrap(); + let chain_spec = ServiceChainSpecInfo { + env: get_env_val(CHAIN_SPEC_ENV, &format!("/{}", filename)), + volume: Volumes::Simple(format!("{}:/{}:ro", spec.display(), filename)), }; + service_config.chain_spec = Some(chain_spec); + } - let mut warnings = Vec::new(); - + // Set up variables + service_config.metrics_port = + cb_config.metrics.as_ref().map(|m| m.start_port).unwrap_or_default(); let needs_signer_module = cb_config.pbs.with_signer || cb_config.modules.as_ref().is_some_and(|modules| { modules.iter().any(|module| matches!(module.kind, ModuleKind::Commit)) }); + let signer_server = if let Some(SignerConfig { inner: SignerType::Remote { url }, .. }) = + &cb_config.signer + { + url.to_string() + } else { + let signer_port = cb_config.signer.as_ref().map(|s| s.port).unwrap_or(SIGNER_PORT_DEFAULT); + format!("http://cb_signer:{signer_port}") + }; // setup modules if let Some(modules_config) = cb_config.modules { @@ -185,77 +249,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re }; // setup pbs service - - let mut pbs_envs = IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT)]); - let mut pbs_volumes = vec![config_volume.clone()]; - - // ports - let host_endpoint = - SocketAddr::from((cb_config.pbs.pbs_config.host, cb_config.pbs.pbs_config.port)); - let mut ports = vec![format!("{}:{}", host_endpoint, cb_config.pbs.pbs_config.port)]; - warnings.push(format!("cb_pbs has an exported port on {}", cb_config.pbs.pbs_config.port)); - - if let Some(mux_config) = cb_config.muxes { - for mux in mux_config.muxes.iter() { - if let Some((env_name, actual_path, internal_path)) = mux.loader_env()? { - let (key, val) = get_env_val(&env_name, &internal_path); - pbs_envs.insert(key, val); - pbs_volumes.push(Volumes::Simple(format!("{actual_path}:{internal_path}:ro"))); - } - } - } - - if let Some((key, val)) = chain_spec_env.clone() { - pbs_envs.insert(key, val); - } - if let Some(metrics_config) = &cb_config.metrics && - metrics_config.enabled - { - let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); - ports.push(format!("{host_endpoint}:{metrics_port}")); - warnings.push(format!("cb_pbs has an exported port on {metrics_port}")); - targets.push(format!("{host_endpoint}")); - let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); - pbs_envs.insert(key, val); - - metrics_port += 1; - } - if log_to_file { - let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); - pbs_envs.insert(key, val); - } - - // inside the container expose on 0.0.0.0 - let container_endpoint = - SocketAddr::from((Ipv4Addr::UNSPECIFIED, cb_config.pbs.pbs_config.port)); - let (key, val) = get_env_val(PBS_ENDPOINT_ENV, &container_endpoint.to_string()); - pbs_envs.insert(key, val); - - // volumes - pbs_volumes.extend(chain_spec_volume.clone()); - pbs_volumes.extend(get_log_volume(&cb_config.logs, CB_MODULE_NAME)); - - let pbs_service = Service { - container_name: Some("cb_pbs".to_owned()), - image: Some(cb_config.pbs.docker_image), - ports: Ports::Short(ports), - volumes: pbs_volumes, - environment: Environment::KvPair(pbs_envs), - healthcheck: Some(Healthcheck { - test: Some(HealthcheckTest::Single(format!( - "curl -f http://localhost:{}{}{}", - cb_config.pbs.pbs_config.port, BUILDER_V1_API_PATH, GET_STATUS_PATH - ))), - interval: Some("30s".into()), - timeout: Some("5s".into()), - retries: 3, - start_interval: None, - start_period: Some("5s".into()), - disable: false, - }), - ..Service::default() - }; - + let pbs_service = create_pbs_service(&mut service_config)?; services.insert("cb_pbs".to_owned(), Some(pbs_service)); // setup signer service @@ -266,240 +260,24 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re match signer_config.inner { SignerType::Local { loader, store } => { - let mut signer_envs = IndexMap::from([ - get_env_val(CONFIG_ENV, CONFIG_DEFAULT), - get_env_same(JWTS_ENV), - ]); - - // Bind the signer API to 0.0.0.0 - let container_endpoint = - SocketAddr::from((Ipv4Addr::UNSPECIFIED, signer_config.port)); - let (key, val) = get_env_val(SIGNER_ENDPOINT_ENV, &container_endpoint.to_string()); - signer_envs.insert(key, val); - - let host_endpoint = SocketAddr::from((signer_config.host, signer_config.port)); - let mut ports = vec![format!("{}:{}", host_endpoint, signer_config.port)]; - warnings.push(format!("cb_signer has an exported port on {}", signer_config.port)); - - if let Some((key, val)) = chain_spec_env.clone() { - signer_envs.insert(key, val); - } - if let Some(metrics_config) = &cb_config.metrics && - metrics_config.enabled - { - let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); - ports.push(format!("{host_endpoint}:{metrics_port}")); - warnings.push(format!("cb_signer has an exported port on {metrics_port}")); - targets.push(format!("{host_endpoint}")); - let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); - signer_envs.insert(key, val); - } - if log_to_file { - let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); - signer_envs.insert(key, val); - } - - // write jwts to env - envs.insert(JWTS_ENV.into(), format_comma_separated(&jwts)); - - // volumes - let mut volumes = vec![config_volume.clone()]; - volumes.extend(chain_spec_volume.clone()); - - match loader { - SignerLoader::File { key_path } => { - volumes.push(Volumes::Simple(format!( - "{}:{}:ro", - key_path.display(), - SIGNER_DEFAULT - ))); - let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_DEFAULT); - signer_envs.insert(k, v); - } - SignerLoader::ValidatorsDir { keys_path, secrets_path, format: _ } => { - volumes.push(Volumes::Simple(format!( - "{}:{}:ro", - keys_path.display(), - SIGNER_DIR_KEYS_DEFAULT - ))); - let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS_DEFAULT); - signer_envs.insert(k, v); - - volumes.push(Volumes::Simple(format!( - "{}:{}:ro", - secrets_path.display(), - SIGNER_DIR_SECRETS_DEFAULT - ))); - let (k, v) = - get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS_DEFAULT); - signer_envs.insert(k, v); - } - }; - - if let Some(store) = store { - match store { - ProxyStore::File { proxy_dir } => { - volumes.push(Volumes::Simple(format!( - "{}:{}:rw", - proxy_dir.display(), - PROXY_DIR_DEFAULT - ))); - let (k, v) = get_env_val(PROXY_DIR_ENV, PROXY_DIR_DEFAULT); - signer_envs.insert(k, v); - } - ProxyStore::ERC2335 { keys_path, secrets_path } => { - volumes.push(Volumes::Simple(format!( - "{}:{}:rw", - keys_path.display(), - PROXY_DIR_KEYS_DEFAULT - ))); - let (k, v) = get_env_val(PROXY_DIR_KEYS_ENV, PROXY_DIR_KEYS_DEFAULT); - signer_envs.insert(k, v); - - volumes.push(Volumes::Simple(format!( - "{}:{}:rw", - secrets_path.display(), - PROXY_DIR_SECRETS_DEFAULT - ))); - let (k, v) = - get_env_val(PROXY_DIR_SECRETS_ENV, PROXY_DIR_SECRETS_DEFAULT); - signer_envs.insert(k, v); - } - } - } - - volumes.extend(get_log_volume(&cb_config.logs, SIGNER_MODULE_NAME)); - - // networks - let signer_networks = vec![SIGNER_NETWORK.to_owned()]; - - let signer_service = Service { - container_name: Some("cb_signer".to_owned()), - image: Some(signer_config.docker_image), - networks: Networks::Simple(signer_networks), - ports: Ports::Short(ports), - volumes, - environment: Environment::KvPair(signer_envs), - healthcheck: Some(Healthcheck { - test: Some(HealthcheckTest::Single(format!( - "curl -f http://localhost:{signer_port}/status" - ))), - interval: Some("30s".into()), - timeout: Some("5s".into()), - retries: 3, - start_interval: None, - start_period: Some("5s".into()), - disable: false, - }), - ..Service::default() - }; - + let signer_service = create_signer_service_local( + &mut service_config, + &signer_config, + &loader, + &store, + )?; services.insert("cb_signer".to_owned(), Some(signer_service)); } SignerType::Dirk { cert_path, key_path, secrets_path, ca_cert_path, store, .. } => { - let mut signer_envs = IndexMap::from([ - get_env_val(CONFIG_ENV, CONFIG_DEFAULT), - get_env_same(JWTS_ENV), - get_env_val(DIRK_CERT_ENV, DIRK_CERT_DEFAULT), - get_env_val(DIRK_KEY_ENV, DIRK_KEY_DEFAULT), - get_env_val(DIRK_DIR_SECRETS_ENV, DIRK_DIR_SECRETS_DEFAULT), - ]); - - // Bind the signer API to 0.0.0.0 - let container_endpoint = - SocketAddr::from((Ipv4Addr::UNSPECIFIED, signer_config.port)); - let (key, val) = get_env_val(SIGNER_ENDPOINT_ENV, &container_endpoint.to_string()); - signer_envs.insert(key, val); - - let host_endpoint = SocketAddr::from((signer_config.host, signer_config.port)); - let mut ports = vec![format!("{}:{}", host_endpoint, signer_config.port)]; - warnings.push(format!("cb_signer has an exported port on {}", signer_config.port)); - - if let Some((key, val)) = chain_spec_env.clone() { - signer_envs.insert(key, val); - } - if let Some(metrics_config) = &cb_config.metrics && - metrics_config.enabled - { - let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); - ports.push(format!("{host_endpoint}:{metrics_port}")); - warnings.push(format!("cb_signer has an exported port on {metrics_port}")); - targets.push(format!("{host_endpoint}")); - let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); - signer_envs.insert(key, val); - } - if log_to_file { - let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); - signer_envs.insert(key, val); - } - - // write jwts to env - envs.insert(JWTS_ENV.into(), format_comma_separated(&jwts)); - - // volumes - let mut volumes = vec![ - config_volume.clone(), - Volumes::Simple(format!("{}:{}:ro", cert_path.display(), DIRK_CERT_DEFAULT)), - Volumes::Simple(format!("{}:{}:ro", key_path.display(), DIRK_KEY_DEFAULT)), - Volumes::Simple(format!( - "{}:{}", - secrets_path.display(), - DIRK_DIR_SECRETS_DEFAULT - )), - ]; - volumes.extend(chain_spec_volume.clone()); - volumes.extend(get_log_volume(&cb_config.logs, SIGNER_MODULE_NAME)); - - if let Some(ca_cert_path) = ca_cert_path { - volumes.push(Volumes::Simple(format!( - "{}:{}:ro", - ca_cert_path.display(), - DIRK_CA_CERT_DEFAULT - ))); - let (key, val) = get_env_val(DIRK_CA_CERT_ENV, DIRK_CA_CERT_DEFAULT); - signer_envs.insert(key, val); - } - - match store { - Some(ProxyStore::File { proxy_dir }) => { - volumes.push(Volumes::Simple(format!( - "{}:{}", - proxy_dir.display(), - PROXY_DIR_DEFAULT - ))); - let (key, val) = get_env_val(PROXY_DIR_ENV, PROXY_DIR_DEFAULT); - signer_envs.insert(key, val); - } - Some(ProxyStore::ERC2335 { .. }) => { - panic!("ERC2335 store not supported with Dirk signer"); - } - None => {} - } - - // networks - let signer_networks = vec![SIGNER_NETWORK.to_owned()]; - - let signer_service = Service { - container_name: Some("cb_signer".to_owned()), - image: Some(signer_config.docker_image), - networks: Networks::Simple(signer_networks), - ports: Ports::Short(ports), - volumes, - environment: Environment::KvPair(signer_envs), - healthcheck: Some(Healthcheck { - test: Some(HealthcheckTest::Single(format!( - "curl -f http://localhost:{signer_port}/status" - ))), - interval: Some("30s".into()), - timeout: Some("5s".into()), - retries: 3, - start_interval: None, - start_period: Some("5s".into()), - disable: false, - }), - ..Service::default() - }; - + let signer_service = create_signer_service_dirk( + &mut service_config, + &signer_config, + &cert_path, + &key_path, + &secrets_path, + &ca_cert_path, + &store, + )?; services.insert("cb_signer".to_owned(), Some(signer_service)); } SignerType::Remote { .. } => { @@ -577,7 +355,348 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re Ok(()) } -fn create_pbs_service() {} +// Creates a PBS service +fn create_pbs_service(service_config: &mut ServiceCreationInfo) -> eyre::Result { + let mut metrics_port = service_config.metrics_port; + let cb_config = &service_config.config_info.cb_config; + let config_volume = &service_config.config_info.config_volume; + let mut envs = IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT)]); + let mut volumes = vec![config_volume.clone()]; + + // Bind the API to 0.0.0.0 + let container_endpoint = + SocketAddr::from((Ipv4Addr::UNSPECIFIED, cb_config.pbs.pbs_config.port)); + let host_endpoint = + SocketAddr::from((cb_config.pbs.pbs_config.host, cb_config.pbs.pbs_config.port)); + let (key, val) = get_env_val(PBS_ENDPOINT_ENV, &container_endpoint.to_string()); + envs.insert(key, val); + + // Exposed ports + let mut ports = vec![format!("{}:{}", host_endpoint, cb_config.pbs.pbs_config.port)]; + service_config + .warnings + .push(format!("cb_pbs has an exported port on {}", cb_config.pbs.pbs_config.port)); + + // Volumes for file-based mux config files + if let Some(ref mux_config) = cb_config.muxes { + for mux in mux_config.muxes.iter() { + if let Some((env_name, actual_path, internal_path)) = mux.loader_env()? { + let (key, val) = get_env_val(&env_name, &internal_path); + envs.insert(key, val); + volumes.push(Volumes::Simple(format!("{actual_path}:{internal_path}:ro"))); + } + } + } + + // Chain spec env/volume + if let Some(spec) = &service_config.chain_spec { + envs.insert(spec.env.0.clone(), spec.env.1.clone()); + volumes.push(spec.volume.clone()); + } + + // Metrics + if let Some(metrics_config) = &cb_config.metrics && + metrics_config.enabled + { + let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); + ports.push(format!("{host_endpoint}:{metrics_port}")); + service_config.warnings.push(format!("cb_pbs has an exported port on {metrics_port}")); + service_config.targets.push(format!("{host_endpoint}")); + let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); + envs.insert(key, val); + + metrics_port += 1; + } + + // Logging + if cb_config.logs.file.enabled { + let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); + envs.insert(key, val); + } + + // Create the service + volumes.extend(get_log_volume(&cb_config.logs, PBS_SERVICE_NAME)); + let pbs_service = Service { + container_name: Some("cb_pbs".to_owned()), + image: Some(cb_config.docker_image.clone()), + ports: Ports::Short(ports), + volumes, + environment: Environment::KvPair(envs), + healthcheck: Some(Healthcheck { + test: Some(HealthcheckTest::Single(format!( + "curl -f http://localhost:{}{}{}", + cb_config.pbs.pbs_config.port, BUILDER_V1_API_PATH, GET_STATUS_PATH + ))), + interval: Some("30s".into()), + timeout: Some("5s".into()), + retries: 3, + start_interval: None, + start_period: Some("5s".into()), + disable: false, + }), + ..Service::default() + }; + + Ok(pbs_service) +} + +// Creates a Signer service using a local signer +fn create_signer_service_local( + service_config: &mut ServiceCreationInfo, + signer_config: &SignerConfig, + loader: &SignerLoader, + store: &Option, +) -> eyre::Result { + let cb_config = &service_config.config_info.cb_config; + let config_volume = &service_config.config_info.config_volume; + let metrics_port = service_config.metrics_port; + let mut envs = + IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT), get_env_same(JWTS_ENV)]); + let mut volumes = vec![config_volume.clone()]; + + // Bind the API to 0.0.0.0 + let container_endpoint = SocketAddr::from((Ipv4Addr::UNSPECIFIED, signer_config.port)); + let host_endpoint = SocketAddr::from((signer_config.host, signer_config.port)); + let (key, val) = get_env_val(SIGNER_ENDPOINT_ENV, &container_endpoint.to_string()); + envs.insert(key, val); + + // Exposed ports + let mut ports = vec![format!("{}:{}", host_endpoint, signer_config.port)]; + service_config + .warnings + .push(format!("cb_signer has an exported port on {}", signer_config.port)); + + // Chain spec env/volume + if let Some(spec) = &service_config.chain_spec { + envs.insert(spec.env.0.clone(), spec.env.1.clone()); + volumes.push(spec.volume.clone()); + } + + // Metrics + if let Some(metrics_config) = &cb_config.metrics && + metrics_config.enabled + { + let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); + ports.push(format!("{host_endpoint}:{metrics_port}")); + service_config.warnings.push(format!("cb_signer has an exported port on {metrics_port}")); + service_config.targets.push(format!("{host_endpoint}")); + let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); + envs.insert(key, val); + service_config.metrics_port += 1; + } + + // Logging + if cb_config.logs.file.enabled { + let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); + envs.insert(key, val); + } + volumes.extend(get_log_volume(&cb_config.logs, SIGNER_SERVICE_NAME)); + + // write jwts to env + service_config.envs.insert(JWTS_ENV.into(), format_comma_separated(&service_config.jwts)); + + // Signer loader volumes and envs + match loader { + SignerLoader::File { key_path } => { + volumes.push(Volumes::Simple(format!("{}:{}:ro", key_path.display(), SIGNER_DEFAULT))); + let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_DEFAULT); + envs.insert(k, v); + } + SignerLoader::ValidatorsDir { keys_path, secrets_path, format: _ } => { + volumes.push(Volumes::Simple(format!( + "{}:{}:ro", + keys_path.display(), + SIGNER_DIR_KEYS_DEFAULT + ))); + let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS_DEFAULT); + envs.insert(k, v); + + volumes.push(Volumes::Simple(format!( + "{}:{}:ro", + secrets_path.display(), + SIGNER_DIR_SECRETS_DEFAULT + ))); + let (k, v) = get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS_DEFAULT); + envs.insert(k, v); + } + }; + + // Proxy keystore volumes and envs + if let Some(store) = store { + match store { + ProxyStore::File { proxy_dir } => { + volumes.push(Volumes::Simple(format!( + "{}:{}:rw", + proxy_dir.display(), + PROXY_DIR_DEFAULT + ))); + let (k, v) = get_env_val(PROXY_DIR_ENV, PROXY_DIR_DEFAULT); + envs.insert(k, v); + } + ProxyStore::ERC2335 { keys_path, secrets_path } => { + volumes.push(Volumes::Simple(format!( + "{}:{}:rw", + keys_path.display(), + PROXY_DIR_KEYS_DEFAULT + ))); + let (k, v) = get_env_val(PROXY_DIR_KEYS_ENV, PROXY_DIR_KEYS_DEFAULT); + envs.insert(k, v); + + volumes.push(Volumes::Simple(format!( + "{}:{}:rw", + secrets_path.display(), + PROXY_DIR_SECRETS_DEFAULT + ))); + let (k, v) = get_env_val(PROXY_DIR_SECRETS_ENV, PROXY_DIR_SECRETS_DEFAULT); + envs.insert(k, v); + } + } + } + + // Create the service + let signer_networks = vec![SIGNER_NETWORK.to_owned()]; + let signer_service = Service { + container_name: Some("cb_signer".to_owned()), + image: Some(cb_config.docker_image.clone()), + networks: Networks::Simple(signer_networks), + ports: Ports::Short(ports), + volumes, + environment: Environment::KvPair(envs), + healthcheck: Some(Healthcheck { + test: Some(HealthcheckTest::Single(format!( + "curl -f http://localhost:{}/status", + signer_config.port, + ))), + interval: Some("30s".into()), + timeout: Some("5s".into()), + retries: 3, + start_interval: None, + start_period: Some("5s".into()), + disable: false, + }), + ..Service::default() + }; + + Ok(signer_service) +} + +// Creates a Signer service that's tied to Dirk +fn create_signer_service_dirk( + service_config: &mut ServiceCreationInfo, + signer_config: &SignerConfig, + cert_path: &Path, + key_path: &Path, + secrets_path: &Path, + ca_cert_path: &Option, + store: &Option, +) -> eyre::Result { + let cb_config = &service_config.config_info.cb_config; + let config_volume = &service_config.config_info.config_volume; + let metrics_port = service_config.metrics_port; + let mut envs = IndexMap::from([ + get_env_val(CONFIG_ENV, CONFIG_DEFAULT), + get_env_same(JWTS_ENV), + get_env_val(DIRK_CERT_ENV, DIRK_CERT_DEFAULT), + get_env_val(DIRK_KEY_ENV, DIRK_KEY_DEFAULT), + get_env_val(DIRK_DIR_SECRETS_ENV, DIRK_DIR_SECRETS_DEFAULT), + ]); + let mut volumes = vec![ + config_volume.clone(), + Volumes::Simple(format!("{}:{}:ro", cert_path.display(), DIRK_CERT_DEFAULT)), + Volumes::Simple(format!("{}:{}:ro", key_path.display(), DIRK_KEY_DEFAULT)), + Volumes::Simple(format!("{}:{}", secrets_path.display(), DIRK_DIR_SECRETS_DEFAULT)), + ]; + + // Bind the API to 0.0.0.0 + let container_endpoint = SocketAddr::from((Ipv4Addr::UNSPECIFIED, signer_config.port)); + let host_endpoint = SocketAddr::from((signer_config.host, signer_config.port)); + let (key, val) = get_env_val(SIGNER_ENDPOINT_ENV, &container_endpoint.to_string()); + envs.insert(key, val); + + // Exposed ports + let mut ports = vec![format!("{}:{}", host_endpoint, signer_config.port)]; + service_config + .warnings + .push(format!("cb_signer has an exported port on {}", signer_config.port)); + + // Chain spec env/volume + if let Some(spec) = &service_config.chain_spec { + envs.insert(spec.env.0.clone(), spec.env.1.clone()); + volumes.push(spec.volume.clone()); + } + + // Metrics + if let Some(metrics_config) = &cb_config.metrics && + metrics_config.enabled + { + let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); + ports.push(format!("{host_endpoint}:{metrics_port}")); + service_config.warnings.push(format!("cb_signer has an exported port on {metrics_port}")); + service_config.targets.push(format!("{host_endpoint}")); + let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); + envs.insert(key, val); + service_config.metrics_port += 1; + } + + // Logging + if cb_config.logs.file.enabled { + let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); + envs.insert(key, val); + } + volumes.extend(get_log_volume(&cb_config.logs, SIGNER_SERVICE_NAME)); + + // write jwts to env + service_config.envs.insert(JWTS_ENV.into(), format_comma_separated(&service_config.jwts)); + + // CA cert volume and env + if let Some(ca_cert_path) = ca_cert_path { + volumes.push(Volumes::Simple(format!( + "{}:{}:ro", + ca_cert_path.display(), + DIRK_CA_CERT_DEFAULT + ))); + let (key, val) = get_env_val(DIRK_CA_CERT_ENV, DIRK_CA_CERT_DEFAULT); + envs.insert(key, val); + } + + // Keystore volumes and envs + match store { + Some(ProxyStore::File { proxy_dir }) => { + volumes.push(Volumes::Simple(format!("{}:{}", proxy_dir.display(), PROXY_DIR_DEFAULT))); + let (key, val) = get_env_val(PROXY_DIR_ENV, PROXY_DIR_DEFAULT); + envs.insert(key, val); + } + Some(ProxyStore::ERC2335 { .. }) => { + panic!("ERC2335 store not supported with Dirk signer"); + } + None => {} + } + + // Create the service + let signer_networks = vec![SIGNER_NETWORK.to_owned()]; + let signer_service = Service { + container_name: Some("cb_signer".to_owned()), + image: Some(cb_config.docker_image.clone()), + networks: Networks::Simple(signer_networks), + ports: Ports::Short(ports), + volumes, + environment: Environment::KvPair(envs), + healthcheck: Some(Healthcheck { + test: Some(HealthcheckTest::Single(format!( + "curl -f http://localhost:{signer_port}/status" + ))), + interval: Some("30s".into()), + timeout: Some("5s".into()), + retries: 3, + start_interval: None, + start_period: Some("5s".into()), + disable: false, + }), + ..Service::default() + }; + + Ok(signer_service) +} /// FOO=${FOO} fn get_env_same(k: &str) -> (String, Option) { diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index bfbed8c1..d792093d 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -22,6 +22,8 @@ pub const PBS_ENDPOINT_ENV: &str = "CB_PBS_ENDPOINT"; pub const MUX_PATH_ENV: &str = "CB_MUX_PATH"; pub const CB_MODULE_NAME: &str = "commit-boost"; +pub const PBS_SERVICE_NAME: &str = "pbs"; +pub const SIGNER_SERVICE_NAME: &str = "signer"; /// Where the signer module should open the server pub const SIGNER_ENDPOINT_ENV: &str = "CB_SIGNER_ENDPOINT"; diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 3a3c23d8..345dc015 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; -use eyre::{Result, bail}; +use docker_image::DockerImage; +use eyre::{Result, bail, ensure}; use serde::{Deserialize, Serialize}; use crate::types::{Chain, ChainLoader, ForkVersion, load_chain_from_file}; @@ -36,6 +37,9 @@ pub struct CommitBoostConfig { pub metrics: Option, #[serde(default)] pub logs: LogsSettings, + /// Docker image for Commit-Boost services + #[serde(default = "default_image")] + pub docker_image: String, } impl CommitBoostConfig { @@ -52,6 +56,13 @@ impl CommitBoostConfig { ) } + // The Docker tag must parse + ensure!(!self.docker_image.is_empty(), "Docker image is empty"); + ensure!( + DockerImage::parse(&self.docker_image).is_ok(), + format!("Invalid Docker image: {}", self.docker_image) + ); + Ok(()) } @@ -107,6 +118,7 @@ impl CommitBoostConfig { signer: helper_config.signer, metrics: helper_config.metrics, logs: helper_config.logs, + docker_image: helper_config.docker_image, }; Ok(config) @@ -146,4 +158,9 @@ struct HelperConfig { metrics: Option, #[serde(default)] logs: LogsSettings, + docker_image: String, +} + +fn default_image() -> String { + CB_IMAGE_DEFAULT.to_string() } diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 0e1848e5..abcc9726 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -10,14 +10,13 @@ use alloy::{ primitives::{U256, utils::format_ether}, providers::{Provider, ProviderBuilder}, }; -use docker_image::DockerImage; use eyre::{Result, ensure}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; use super::{ CommitBoostConfig, HTTP_TIMEOUT_SECONDS_DEFAULT, PBS_ENDPOINT_ENV, RuntimeMuxConfig, - constants::CB_IMAGE_DEFAULT, load_optional_env_var, + load_optional_env_var, }; use crate::{ commit::client::SignerClient, @@ -204,9 +203,6 @@ impl PbsConfig { /// Static pbs config from config file #[derive(Debug, Deserialize, Serialize)] pub struct StaticPbsConfig { - /// Docker image of the module - #[serde(default = "default_image")] - pub docker_image: String, /// Config of pbs module #[serde(flatten)] pub pbs_config: PbsConfig, @@ -220,13 +216,6 @@ impl StaticPbsConfig { pub async fn validate(&self, chain: Chain) -> Result<()> { self.pbs_config.validate(chain).await?; - // The Docker tag must parse - ensure!(!self.docker_image.is_empty(), "Docker image is empty"); - ensure!( - DockerImage::parse(&self.docker_image).is_ok(), - format!("Invalid Docker image: {}", self.docker_image) - ); - Ok(()) } } @@ -254,10 +243,6 @@ pub struct PbsModuleConfig { pub mux_lookup: Option>, } -fn default_image() -> String { - CB_IMAGE_DEFAULT.to_string() -} - /// Loads the default pbs config, i.e. with no signer client or custom data pub async fn load_pbs_config() -> Result { let config = CommitBoostConfig::from_env_path()?; @@ -344,7 +329,7 @@ pub async fn load_pbs_custom_config() -> Result<(PbsModuleC // load module config including the extra data (if any) let cb_config: StubConfig = load_file_from_env(CONFIG_ENV)?; - cb_config.pbs.static_config.pbs_config.validate(cb_config.chain).await?; + cb_config.pbs.static_config.validate(cb_config.chain).await?; // use endpoint from env if set, otherwise use default host and port let endpoint = if let Some(endpoint) = load_optional_env_var(PBS_ENDPOINT_ENV) { diff --git a/examples/status_api/src/main.rs b/examples/status_api/src/main.rs index 7ad9b533..e5260b37 100644 --- a/examples/status_api/src/main.rs +++ b/examples/status_api/src/main.rs @@ -91,7 +91,7 @@ async fn main() -> Result<()> { let (pbs_config, extra) = load_pbs_custom_config::().await?; let chain = pbs_config.chain; - let _guard = initialize_tracing_log(PBS_MODULE_NAME, LogsSettings::from_env_config()?)?; + let _guard = initialize_tracing_log(PBS_SERVICE_NAME, LogsSettings::from_env_config()?)?; let custom_state = MyBuilderState::from_config(extra); let state = PbsState::new(pbs_config).with_data(custom_state); From f57d41b4633e175e43995ae3462d2ca25bcc73d5 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Mon, 2 Feb 2026 11:04:01 -0500 Subject: [PATCH 03/22] More refactoring of handle_docker_init --- config.example.toml | 7 +- crates/cli/src/docker_init.rs | 380 +++++++++++++++-------------- crates/common/src/config/module.rs | 4 +- tests/tests/config.rs | 2 +- 4 files changed, 209 insertions(+), 184 deletions(-) diff --git a/config.example.toml b/config.example.toml index e85b98e1..ee8f0909 100644 --- a/config.example.toml +++ b/config.example.toml @@ -7,11 +7,12 @@ # A custom object, e.g., chain = { genesis_time_secs = 1695902400, slot_time_secs = 12, genesis_fork_version = "0x01017000" }. chain = "Holesky" +# Docker image to use for the Commit-Boost services. +# OPTIONAL, DEFAULT: ghcr.io/commit-boost/cb:latest +docker_image = "ghcr.io/commit-boost/cb:latest" + # Configuration for the PBS module [pbs] -# Docker image to use for the PBS module. -# OPTIONAL, DEFAULT: ghcr.io/commit-boost/pbs:latest -docker_image = "ghcr.io/commit-boost/pbs:latest" # Whether to enable the PBS module to request signatures from the Signer module (not used in the default PBS image) # OPTIONAL, DEFAULT: false with_signer = false diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index a65e6740..0f0bc465 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -6,15 +6,16 @@ use std::{ use cb_common::{ config::{ - CB_MODULE_NAME, CHAIN_SPEC_ENV, CONFIG_DEFAULT, CONFIG_ENV, CommitBoostConfig, - DIRK_CA_CERT_DEFAULT, DIRK_CA_CERT_ENV, DIRK_CERT_DEFAULT, DIRK_CERT_ENV, - DIRK_DIR_SECRETS_DEFAULT, DIRK_DIR_SECRETS_ENV, DIRK_KEY_DEFAULT, DIRK_KEY_ENV, JWTS_ENV, - LOGS_DIR_DEFAULT, LOGS_DIR_ENV, LogsSettings, METRICS_PORT_ENV, MODULE_ID_ENV, - MODULE_JWT_ENV, ModuleKind, PBS_ENDPOINT_ENV, PBS_SERVICE_NAME, PROXY_DIR_DEFAULT, - PROXY_DIR_ENV, PROXY_DIR_KEYS_DEFAULT, PROXY_DIR_KEYS_ENV, PROXY_DIR_SECRETS_DEFAULT, + CHAIN_SPEC_ENV, CONFIG_DEFAULT, CONFIG_ENV, CommitBoostConfig, DIRK_CA_CERT_DEFAULT, + DIRK_CA_CERT_ENV, DIRK_CERT_DEFAULT, DIRK_CERT_ENV, DIRK_DIR_SECRETS_DEFAULT, + DIRK_DIR_SECRETS_ENV, DIRK_KEY_DEFAULT, DIRK_KEY_ENV, JWTS_ENV, LOGS_DIR_DEFAULT, + LOGS_DIR_ENV, LogsSettings, METRICS_PORT_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, ModuleKind, + PBS_ENDPOINT_ENV, PBS_SERVICE_NAME, PROXY_DIR_DEFAULT, PROXY_DIR_ENV, + PROXY_DIR_KEYS_DEFAULT, PROXY_DIR_KEYS_ENV, PROXY_DIR_SECRETS_DEFAULT, PROXY_DIR_SECRETS_ENV, SIGNER_DEFAULT, SIGNER_DIR_KEYS_DEFAULT, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS_DEFAULT, SIGNER_DIR_SECRETS_ENV, SIGNER_ENDPOINT_ENV, SIGNER_KEYS_ENV, SIGNER_PORT_DEFAULT, SIGNER_SERVICE_NAME, SIGNER_URL_ENV, SignerConfig, SignerType, + StaticModuleConfig, }, pbs::{BUILDER_V1_API_PATH, GET_STATUS_PATH}, signer::{ProxyStore, SignerLoader}, @@ -47,9 +48,6 @@ struct ServiceChainSpecInfo { // Info about the Commit-Boost config being used to create services struct CommitBoostConfigInfo { - // Path to the config file - config_path: PathBuf, - // Commit-Boost config cb_config: CommitBoostConfig, @@ -83,33 +81,20 @@ struct ServiceCreationInfo { metrics_port: u16, } -// Information about the created CB Signer service -struct SignerService { - // The created signer, as a docker compose service - service: Service, - - // The port used for metrics - metrics_port: u16, - - // The port used for signer API communication - signer_port: u16, -} - /// Builds the docker compose file for the Commit-Boost services // TODO: do more validation for paths, images, etc pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Result<()> { + // Initialize variables let mut services = IndexMap::new(); - println!("Initializing Commit-Boost with config file: {}", config_path.display()); let mut service_config = ServiceCreationInfo { config_info: CommitBoostConfigInfo { - config_path: config_path.clone(), - cb_config: CommitBoostConfig::from_file(&config_path)?, config_volume: Volumes::Simple(format!( "./{}:{}:ro", config_path.display(), CONFIG_DEFAULT )), + cb_config: CommitBoostConfig::from_file(&config_path)?, }, envs: IndexMap::new(), targets: Vec::new(), @@ -118,10 +103,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re chain_spec: None, metrics_port: 9100, }; - - // Validate the CB config - let cb_config = &mut service_config.config_info.cb_config; - cb_config.validate().await?; + service_config.config_info.cb_config.validate().await?; // Get the custom chain spec, if any let chain_spec_path = CommitBoostConfig::chain_spec_file(&config_path); @@ -136,114 +118,45 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re } // Set up variables - service_config.metrics_port = - cb_config.metrics.as_ref().map(|m| m.start_port).unwrap_or_default(); - let needs_signer_module = cb_config.pbs.with_signer || - cb_config.modules.as_ref().is_some_and(|modules| { + service_config.metrics_port = service_config + .config_info + .cb_config + .metrics + .as_ref() + .map(|m| m.start_port) + .unwrap_or_default(); + let needs_signer_module = service_config.config_info.cb_config.pbs.with_signer || + service_config.config_info.cb_config.modules.as_ref().is_some_and(|modules| { modules.iter().any(|module| matches!(module.kind, ModuleKind::Commit)) }); + let signer_config = + if needs_signer_module { + Some(service_config.config_info.cb_config.signer.clone().expect( + "Signer module required but no signer config provided in Commit-Boost config", + )) + } else { + None + }; let signer_server = if let Some(SignerConfig { inner: SignerType::Remote { url }, .. }) = - &cb_config.signer + &service_config.config_info.cb_config.signer { url.to_string() } else { - let signer_port = cb_config.signer.as_ref().map(|s| s.port).unwrap_or(SIGNER_PORT_DEFAULT); + let signer_port = service_config + .config_info + .cb_config + .signer + .as_ref() + .map(|s| s.port) + .unwrap_or(SIGNER_PORT_DEFAULT); format!("http://cb_signer:{signer_port}") }; // setup modules - if let Some(modules_config) = cb_config.modules { - for module in modules_config { - let module_cid = format!("cb_{}", module.id.to_lowercase()); - - let module_service = match module.kind { - // a commit module needs a JWT and access to the signer network - ModuleKind::Commit => { - let mut ports = vec![]; - - let jwt_secret = random_jwt_secret(); - let jwt_name = format!("CB_JWT_{}", module.id.to_uppercase()); - - // module ids are assumed unique, so envs dont override each other - let mut module_envs = IndexMap::from([ - get_env_val(MODULE_ID_ENV, &module.id), - get_env_val(CONFIG_ENV, CONFIG_DEFAULT), - get_env_interp(MODULE_JWT_ENV, &jwt_name), - get_env_val(SIGNER_URL_ENV, &signer_server), - ]); - - // Pass on the env variables - if let Some(envs) = module.env { - for (k, v) in envs { - module_envs.insert(k, Some(SingleValue::String(v))); - } - } - - // Set environment file - let env_file = module.env_file.map(EnvFile::Simple); - - if let Some((key, val)) = chain_spec_env.clone() { - module_envs.insert(key, val); - } - - if let Some(metrics_config) = &cb_config.metrics && - metrics_config.enabled - { - let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); - ports.push(format!("{host_endpoint}:{metrics_port}")); - warnings - .push(format!("{module_cid} has an exported port on {metrics_port}")); - targets.push(format!("{host_endpoint}")); - let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); - module_envs.insert(key, val); - - metrics_port += 1; - } - - if log_to_file { - let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); - module_envs.insert(key, val); - } - - envs.insert(jwt_name.clone(), jwt_secret.clone()); - jwts.insert(module.id.clone(), jwt_secret); - - // networks - let module_networks = vec![SIGNER_NETWORK.to_owned()]; - - // volumes - let mut module_volumes = vec![config_volume.clone()]; - module_volumes.extend(chain_spec_volume.clone()); - module_volumes.extend(get_log_volume(&cb_config.logs, &module.id)); - - // depends_on - let mut module_dependencies = IndexMap::new(); - module_dependencies.insert("cb_signer".into(), DependsCondition { - condition: "service_healthy".into(), - }); - - Service { - container_name: Some(module_cid.clone()), - image: Some(module.docker_image), - networks: Networks::Simple(module_networks), - ports: Ports::Short(ports), - volumes: module_volumes, - environment: Environment::KvPair(module_envs), - depends_on: if let Some(SignerConfig { - inner: SignerType::Remote { .. }, - .. - }) = &cb_config.signer - { - DependsOnOptions::Simple(vec![]) - } else { - DependsOnOptions::Conditional(module_dependencies) - }, - env_file, - ..Service::default() - } - } - }; - + if let Some(ref modules_config) = service_config.config_info.cb_config.modules { + for module in modules_config.clone() { + let (module_cid, module_service) = + create_module_service(&module, signer_server.as_str(), &mut service_config)?; services.insert(module_cid, Some(module_service)); } }; @@ -253,18 +166,14 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re services.insert("cb_pbs".to_owned(), Some(pbs_service)); // setup signer service - if needs_signer_module { - let Some(signer_config) = cb_config.signer else { - panic!("Signer module required but no signer config provided"); - }; - - match signer_config.inner { + if let Some(signer_config) = signer_config { + match &signer_config.inner { SignerType::Local { loader, store } => { let signer_service = create_signer_service_local( &mut service_config, &signer_config, - &loader, - &store, + loader, + store, )?; services.insert("cb_signer".to_owned(), Some(signer_service)); } @@ -272,11 +181,11 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re let signer_service = create_signer_service_dirk( &mut service_config, &signer_config, - &cert_path, - &key_path, - &secrets_path, - &ca_cert_path, - &store, + cert_path, + key_path, + secrets_path, + ca_cert_path, + store, )?; services.insert("cb_signer".to_owned(), Some(signer_service)); } @@ -298,55 +207,26 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re ); } - compose.services = Services(services); - // write compose to file - let compose_str = serde_yaml::to_string(&compose)?; + compose.services = Services(services); let compose_path = Path::new(&output_dir).join(CB_COMPOSE_FILE); - std::fs::write(&compose_path, compose_str)?; - if !warnings.is_empty() { - println!(); - for exposed_port in warnings { - println!("Warning: {exposed_port}"); - } - println!() - } - // if file logging is enabled, warn about permissions - if cb_config.logs.file.enabled { - let log_dir = cb_config.logs.file.dir_path; - println!( - "Warning: file logging is enabled, you may need to update permissions for the logs directory. e.g. with:\n\t`sudo chown -R 10001:10001 {}`", - log_dir.display() - ); - println!() - } - - println!("Docker Compose file written to: {compose_path:?}"); + write_compose_file(&compose, &compose_path, &service_config)?; - // write prometheus targets to file - if !targets.is_empty() { - let targets = targets.join(", "); + // Inform user about Prometheus targets + if !service_config.targets.is_empty() { + let targets = service_config.targets.join(", "); println!("Note: Make sure to add these targets for Prometheus to scrape: {targets}"); println!( "Check out the docs on how to configure Prometheus/Grafana/cAdvisor: https://commit-boost.github.io/commit-boost-client/get_started/running/metrics" ); } - if envs.is_empty() { + if service_config.envs.is_empty() { println!("Run with:\n\tdocker compose -f {compose_path:?} up -d"); } else { // write envs to .env file - let envs_str = { - let mut envs_str = String::new(); - for (k, v) in envs { - envs_str.push_str(&format!("{k}={v}\n")); - } - envs_str - }; let env_path = Path::new(&output_dir).join(CB_ENV_FILE); - std::fs::write(&env_path, envs_str)?; - println!("Env file written to: {env_path:?}"); - + write_env_file(&service_config.envs, &env_path)?; println!(); println!("Run with:\n\tdocker compose --env-file {env_path:?} -f {compose_path:?} up -d"); println!("Stop with:\n\tdocker compose --env-file {env_path:?} -f {compose_path:?} down"); @@ -357,7 +237,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re // Creates a PBS service fn create_pbs_service(service_config: &mut ServiceCreationInfo) -> eyre::Result { - let mut metrics_port = service_config.metrics_port; + let metrics_port = service_config.metrics_port; let cb_config = &service_config.config_info.cb_config; let config_volume = &service_config.config_info.config_volume; let mut envs = IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT)]); @@ -405,7 +285,7 @@ fn create_pbs_service(service_config: &mut ServiceCreationInfo) -> eyre::Result< let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); envs.insert(key, val); - metrics_port += 1; + service_config.metrics_port += 1; } // Logging @@ -683,7 +563,8 @@ fn create_signer_service_dirk( environment: Environment::KvPair(envs), healthcheck: Some(Healthcheck { test: Some(HealthcheckTest::Single(format!( - "curl -f http://localhost:{signer_port}/status" + "curl -f http://localhost:{}/status", + signer_config.port, ))), interval: Some("30s".into()), timeout: Some("5s".into()), @@ -698,6 +579,149 @@ fn create_signer_service_dirk( Ok(signer_service) } +/// Creates a Commit-Boost module service +fn create_module_service( + module: &StaticModuleConfig, + signer_server: &str, + service_config: &mut ServiceCreationInfo, +) -> eyre::Result<(String, Service)> { + let cb_config = &service_config.config_info.cb_config; + let config_volume = &service_config.config_info.config_volume; + let metrics_port = service_config.metrics_port; + let module_cid = format!("cb_{}", module.id.to_lowercase()); + + let module_service = match module.kind { + // a commit module needs a JWT and access to the signer network + ModuleKind::Commit => { + let mut ports = vec![]; + + let jwt_secret = random_jwt_secret(); + let jwt_name = format!("CB_JWT_{}", module.id.to_uppercase()); + + // module ids are assumed unique, so envs dont override each other + let mut module_envs = IndexMap::from([ + get_env_val(MODULE_ID_ENV, &module.id), + get_env_val(CONFIG_ENV, CONFIG_DEFAULT), + get_env_interp(MODULE_JWT_ENV, &jwt_name), + get_env_val(SIGNER_URL_ENV, signer_server), + ]); + + // Pass on the env variables + if let Some(envs) = &module.env { + for (k, v) in envs { + module_envs.insert(k.clone(), Some(SingleValue::String(v.clone()))); + } + }; + + // volumes + let mut module_volumes = vec![config_volume.clone()]; + module_volumes.extend(get_log_volume(&cb_config.logs, &module.id)); + + // Chain spec env/volume + if let Some(spec) = &service_config.chain_spec { + module_envs.insert(spec.env.0.clone(), spec.env.1.clone()); + module_volumes.push(spec.volume.clone()); + } + + if let Some(metrics_config) = &cb_config.metrics && + metrics_config.enabled + { + let host_endpoint = SocketAddr::from((metrics_config.host, metrics_port)); + ports.push(format!("{host_endpoint}:{metrics_port}")); + service_config + .warnings + .push(format!("{module_cid} has an exported port on {metrics_port}")); + service_config.targets.push(format!("{host_endpoint}")); + let (key, val) = get_env_uval(METRICS_PORT_ENV, metrics_port as u64); + module_envs.insert(key, val); + + service_config.metrics_port += 1; + } + + // Logging + if cb_config.logs.file.enabled { + let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); + module_envs.insert(key, val); + } + + // write jwts to env + service_config.envs.insert(jwt_name.clone(), jwt_secret.clone()); + service_config.jwts.insert(module.id.clone(), jwt_secret); + + // Dependencies + let mut module_dependencies = IndexMap::new(); + module_dependencies.insert("cb_signer".into(), DependsCondition { + condition: "service_healthy".into(), + }); + + // Create the service + let module_networks = vec![SIGNER_NETWORK.to_owned()]; + Service { + container_name: Some(module_cid.clone()), + image: Some(module.docker_image.clone()), + networks: Networks::Simple(module_networks), + ports: Ports::Short(ports), + volumes: module_volumes, + environment: Environment::KvPair(module_envs), + depends_on: if let Some(SignerConfig { inner: SignerType::Remote { .. }, .. }) = + &cb_config.signer + { + DependsOnOptions::Simple(vec![]) + } else { + DependsOnOptions::Conditional(module_dependencies) + }, + env_file: module.env_file.clone().map(EnvFile::Simple), + ..Service::default() + } + } + }; + + Ok((module_cid, module_service)) +} + +/// Writes the docker compose file to disk and prints any warnings +fn write_compose_file( + compose: &Compose, + output_path: &Path, + service_config: &ServiceCreationInfo, +) -> Result<()> { + let compose_str = serde_yaml::to_string(compose)?; + std::fs::write(output_path, compose_str)?; + if !service_config.warnings.is_empty() { + println!(); + for exposed_port in &service_config.warnings { + println!("Warning: {exposed_port}"); + } + println!() + } + // if file logging is enabled, warn about permissions + let cb_config = &service_config.config_info.cb_config; + if cb_config.logs.file.enabled { + let log_dir = &cb_config.logs.file.dir_path; + println!( + "Warning: file logging is enabled, you may need to update permissions for the logs directory. e.g. with:\n\t`sudo chown -R 10001:10001 {}`", + log_dir.display() + ); + println!() + } + println!("Docker Compose file written to: {output_path:?}"); + Ok(()) +} + +/// Writes the envs to a .env file +fn write_env_file(envs: &IndexMap, output_path: &Path) -> Result<()> { + let envs_str = { + let mut envs_str = String::new(); + for (k, v) in envs { + envs_str.push_str(&format!("{k}={v}\n")); + } + envs_str + }; + std::fs::write(output_path, envs_str)?; + println!("Env file written to: {output_path:?}"); + Ok(()) +} + /// FOO=${FOO} fn get_env_same(k: &str) -> (String, Option) { get_env_interp(k, k) diff --git a/crates/common/src/config/module.rs b/crates/common/src/config/module.rs index 02fa90da..7c5b52c3 100644 --- a/crates/common/src/config/module.rs +++ b/crates/common/src/config/module.rs @@ -14,14 +14,14 @@ use crate::{ types::{Chain, Jwt, ModuleId}, }; -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub enum ModuleKind { #[serde(alias = "commit")] Commit, } /// Static module config from config file -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct StaticModuleConfig { /// Unique id of the module pub id: ModuleId, diff --git a/tests/tests/config.rs b/tests/tests/config.rs index bffefcbc..3d03f7ca 100644 --- a/tests/tests/config.rs +++ b/tests/tests/config.rs @@ -36,7 +36,7 @@ async fn test_load_pbs_happy() -> Result<()> { ); // Docker and general settings - assert_eq!(config.pbs.docker_image, "ghcr.io/commit-boost/pbs:latest"); + assert_eq!(config.docker_image, "ghcr.io/commit-boost/cb:latest"); assert!(!config.pbs.with_signer); assert_eq!(config.pbs.pbs_config.host, "127.0.0.1".parse::().unwrap()); assert_eq!(config.pbs.pbs_config.port, 18550); From 4d75ddc32eabd4a6665b2958d57f7c559e2ab403 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Mon, 2 Feb 2026 14:57:44 -0500 Subject: [PATCH 04/22] Fixed some tests --- config.example.toml | 4 ++-- tests/tests/config.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.example.toml b/config.example.toml index ee8f0909..445eb4ed 100644 --- a/config.example.toml +++ b/config.example.toml @@ -8,8 +8,8 @@ chain = "Holesky" # Docker image to use for the Commit-Boost services. -# OPTIONAL, DEFAULT: ghcr.io/commit-boost/cb:latest -docker_image = "ghcr.io/commit-boost/cb:latest" +# OPTIONAL, DEFAULT: ghcr.io/commit-boost/commit-boost:latest +docker_image = "ghcr.io/commit-boost/commit-boost:latest" # Configuration for the PBS module [pbs] diff --git a/tests/tests/config.rs b/tests/tests/config.rs index 3d03f7ca..1f4f6b4d 100644 --- a/tests/tests/config.rs +++ b/tests/tests/config.rs @@ -36,7 +36,7 @@ async fn test_load_pbs_happy() -> Result<()> { ); // Docker and general settings - assert_eq!(config.docker_image, "ghcr.io/commit-boost/cb:latest"); + assert_eq!(config.docker_image, "ghcr.io/commit-boost/commit-boost:latest"); assert!(!config.pbs.with_signer); assert_eq!(config.pbs.pbs_config.host, "127.0.0.1".parse::().unwrap()); assert_eq!(config.pbs.pbs_config.port, 18550); From 19d0efed551fe6253d1047ed8a8dc952932e1eb6 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Mon, 2 Feb 2026 22:35:02 -0500 Subject: [PATCH 05/22] Reverted to multiple Docker images, merged CLI binary into CB --- .github/workflows/release.yml | 9 ++-- bin/Cargo.toml | 4 -- bin/commit-boost.rs | 63 ++++++++++++++++++++++++--- bin/signer.rs | 46 ------------------- config.example.toml | 7 ++- crates/cli/src/docker_init.rs | 6 +-- crates/cli/src/lib.rs | 46 +------------------ crates/common/src/config/constants.rs | 10 +++-- crates/common/src/config/mod.rs | 19 +------- crates/common/src/config/pbs.rs | 26 ++++++++--- crates/common/src/config/signer.rs | 20 ++++++++- justfile | 62 +++++++++++--------------- provisioning/pbs.Dockerfile | 5 ++- provisioning/signer.Dockerfile | 5 ++- tests/src/utils.rs | 7 +-- tests/tests/config.rs | 2 +- 16 files changed, 151 insertions(+), 186 deletions(-) delete mode 100644 bin/signer.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be779b85..69725a28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,8 +19,7 @@ jobs: - arm64 name: - commit-boost-cli - - commit-boost-pbs - - commit-boost-signer + - commit-boost include: - target: amd64 package-suffix: x86-64 @@ -28,10 +27,8 @@ jobs: package-suffix: arm64 - name: commit-boost-cli target-crate: cli - - name: commit-boost-pbs - target-crate: pbs - - name: commit-boost-signer - target-crate: signer + - name: commit-boost + target-crate: commit-boost runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 0871c67b..19744c1f 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -18,10 +18,6 @@ tracing.workspace = true tree_hash.workspace = true tree_hash_derive.workspace = true -[[bin]] -name = "commit-boost-cli" -path = "cli.rs" - [[bin]] name = "commit-boost" path = "commit-boost.rs" diff --git a/bin/commit-boost.rs b/bin/commit-boost.rs index 3078a6f9..89f1a27e 100644 --- a/bin/commit-boost.rs +++ b/bin/commit-boost.rs @@ -1,8 +1,14 @@ +use std::path::PathBuf; + +use cb_cli::docker_init::handle_docker_init; use cb_common::{ - config::{LogsSettings, PBS_SERVICE_NAME, load_pbs_config}, - utils::{initialize_tracing_log, wait_for_signal}, + config::{ + LogsSettings, PBS_SERVICE_NAME, SIGNER_SERVICE_NAME, StartSignerConfig, load_pbs_config, + }, + utils::{initialize_tracing_log, print_logo, wait_for_signal}, }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; +use cb_signer::service::SigningService; use clap::{Parser, Subcommand}; use eyre::Result; use tracing::{error, info}; @@ -10,9 +16,12 @@ use tracing::{error, info}; /// Version string with a leading 'v' const VERSION: &str = concat!("v", env!("CARGO_PKG_VERSION")); +/// Long about string for the CLI +const LONG_ABOUT: &str = "Commit-Boost allows Ethereum validators to safely run MEV-Boost and community-built commitment protocols"; + /// Subcommands and global arguments for the module #[derive(Parser, Debug)] -#[command(name = "Commit-Boost PBS Service", version = VERSION, about, long_about = None)] +#[command(name = "Commit-Boost", version = VERSION, about, long_about = LONG_ABOUT)] struct Cli { #[command(subcommand)] command: Commands, @@ -25,6 +34,17 @@ enum Commands { /// Run the Signer service Signer, + + /// Generate the starting docker-compose file (the old CLI binary) + Init { + /// Path to config file + #[arg(long("config"))] + config_path: PathBuf, + + /// Path to output files + #[arg(short, long("output"), default_value = "./")] + output_path: PathBuf, + }, } #[tokio::main] @@ -35,10 +55,18 @@ async fn main() -> Result<()> { color_eyre::install()?; - let _guard = initialize_tracing_log(PBS_SERVICE_NAME, LogsSettings::from_env_config()?); + match _cli.command { + Commands::Pbs => run_pbs_service().await?, + Commands::Signer => run_signer_service().await?, + Commands::Init { config_path, output_path } => run_init(config_path, output_path).await?, + } - let _args = cb_cli::CbArgs::parse(); + Ok(()) +} +/// Run the PBS service +async fn run_pbs_service() -> Result<()> { + let _guard = initialize_tracing_log(PBS_SERVICE_NAME, LogsSettings::from_env_config()?); let pbs_config = load_pbs_config().await?; PbsService::init_metrics(pbs_config.chain)?; @@ -56,6 +84,31 @@ async fn main() -> Result<()> { info!("shutting down"); } } + Ok(()) +} + +/// Run the Signer service +async fn run_signer_service() -> Result<()> { + let _guard = initialize_tracing_log(SIGNER_SERVICE_NAME, LogsSettings::from_env_config()?); + let config = StartSignerConfig::load_from_env()?; + let server = SigningService::run(config); + + tokio::select! { + maybe_err = server => { + if let Err(err) = maybe_err { + error!(%err, "signing server unexpectedly stopped"); + eprintln!("signing server unexpectedly stopped: {err}"); + } + }, + _ = wait_for_signal() => { + info!("shutting down"); + } + } Ok(()) } + +async fn run_init(config_path: PathBuf, output_path: PathBuf) -> Result<()> { + print_logo(); + handle_docker_init(config_path, output_path).await +} diff --git a/bin/signer.rs b/bin/signer.rs deleted file mode 100644 index 01f3c970..00000000 --- a/bin/signer.rs +++ /dev/null @@ -1,46 +0,0 @@ -use cb_common::{ - config::{LogsSettings, SIGNER_MODULE_NAME, StartSignerConfig}, - utils::{initialize_tracing_log, wait_for_signal}, -}; -use cb_signer::service::SigningService; -use clap::Parser; -use eyre::Result; -use tracing::{error, info}; - -/// Version string with a leading 'v' -const VERSION: &str = concat!("v", env!("CARGO_PKG_VERSION")); - -/// Subcommands and global arguments for the module -#[derive(Parser, Debug)] -#[command(name = "Commit-Boost Signer Service", version = VERSION, about, long_about = None)] -struct Cli {} - -#[tokio::main] -async fn main() -> Result<()> { - // Parse the CLI arguments (currently only used for version info, more can be - // added later) - let _cli = Cli::parse(); - - color_eyre::install()?; - - let _guard = initialize_tracing_log(SIGNER_MODULE_NAME, LogsSettings::from_env_config()?); - - let _args = cb_cli::SignerArgs::parse(); - - let config = StartSignerConfig::load_from_env()?; - let server = SigningService::run(config); - - tokio::select! { - maybe_err = server => { - if let Err(err) = maybe_err { - error!(%err, "signing server unexpectedly stopped"); - eprintln!("signing server unexpectedly stopped: {err}"); - } - }, - _ = wait_for_signal() => { - info!("shutting down"); - } - } - - Ok(()) -} diff --git a/config.example.toml b/config.example.toml index 445eb4ed..e85b98e1 100644 --- a/config.example.toml +++ b/config.example.toml @@ -7,12 +7,11 @@ # A custom object, e.g., chain = { genesis_time_secs = 1695902400, slot_time_secs = 12, genesis_fork_version = "0x01017000" }. chain = "Holesky" -# Docker image to use for the Commit-Boost services. -# OPTIONAL, DEFAULT: ghcr.io/commit-boost/commit-boost:latest -docker_image = "ghcr.io/commit-boost/commit-boost:latest" - # Configuration for the PBS module [pbs] +# Docker image to use for the PBS module. +# OPTIONAL, DEFAULT: ghcr.io/commit-boost/pbs:latest +docker_image = "ghcr.io/commit-boost/pbs:latest" # Whether to enable the PBS module to request signatures from the Signer module (not used in the default PBS image) # OPTIONAL, DEFAULT: false with_signer = false diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 0f0bc465..ada3241d 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -298,7 +298,7 @@ fn create_pbs_service(service_config: &mut ServiceCreationInfo) -> eyre::Result< volumes.extend(get_log_volume(&cb_config.logs, PBS_SERVICE_NAME)); let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), - image: Some(cb_config.docker_image.clone()), + image: Some(cb_config.pbs.docker_image.clone()), ports: Ports::Short(ports), volumes, environment: Environment::KvPair(envs), @@ -437,7 +437,7 @@ fn create_signer_service_local( let signer_networks = vec![SIGNER_NETWORK.to_owned()]; let signer_service = Service { container_name: Some("cb_signer".to_owned()), - image: Some(cb_config.docker_image.clone()), + image: Some(signer_config.docker_image.clone()), networks: Networks::Simple(signer_networks), ports: Ports::Short(ports), volumes, @@ -556,7 +556,7 @@ fn create_signer_service_dirk( let signer_networks = vec![SIGNER_NETWORK.to_owned()]; let signer_service = Service { container_name: Some("cb_signer".to_owned()), - image: Some(cb_config.docker_image.clone()), + image: Some(signer_config.docker_image.clone()), networks: Networks::Simple(signer_networks), ports: Ports::Short(ports), volumes, diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index f14ab11f..7257d53b 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,45 +1 @@ -use std::path::PathBuf; - -use cb_common::utils::print_logo; -use clap::{Parser, Subcommand}; - -mod docker_init; - -#[derive(Parser, Debug)] -#[command(version, about, long_about = LONG_ABOUT, name = "commit-boost-cli")] -pub struct CliArgs { - #[command(subcommand)] - pub cmd: Command, -} - -#[derive(Debug, Subcommand)] -pub enum Command { - /// Generate the starting docker-compose file - Init { - /// Path to config file - #[arg(long("config"))] - config_path: PathBuf, - - /// Path to output files - #[arg(short, long("output"), default_value = "./")] - output_path: PathBuf, - }, -} - -impl CliArgs { - pub async fn run(self) -> eyre::Result<()> { - print_logo(); - - match self.cmd { - Command::Init { config_path, output_path } => { - docker_init::handle_docker_init(config_path, output_path).await - } - } - } -} - -const LONG_ABOUT: &str = "Commit-Boost allows Ethereum validators to safely run MEV-Boost and community-built commitment protocols"; - -#[derive(Parser, Debug)] -#[command(version, about, long_about = LONG_ABOUT, name = "commit-boost")] -pub struct CbArgs; +pub mod docker_init; diff --git a/crates/common/src/config/constants.rs b/crates/common/src/config/constants.rs index d792093d..a2942f3a 100644 --- a/crates/common/src/config/constants.rs +++ b/crates/common/src/config/constants.rs @@ -14,15 +14,19 @@ pub const METRICS_PORT_ENV: &str = "CB_METRICS_PORT"; pub const LOGS_DIR_ENV: &str = "CB_LOGS_DIR"; pub const LOGS_DIR_DEFAULT: &str = "/var/logs/commit-boost"; -pub const CB_IMAGE_DEFAULT: &str = "ghcr.io/commit-boost/commit-boost:latest"; +///////////////////////// PBS ///////////////////////// + +pub const PBS_IMAGE_DEFAULT: &str = "ghcr.io/commit-boost/pbs:latest"; +pub const PBS_SERVICE_NAME: &str = "pbs"; /// Where to receive BuilderAPI calls from beacon node pub const PBS_ENDPOINT_ENV: &str = "CB_PBS_ENDPOINT"; pub const MUX_PATH_ENV: &str = "CB_MUX_PATH"; -pub const CB_MODULE_NAME: &str = "commit-boost"; -pub const PBS_SERVICE_NAME: &str = "pbs"; +///////////////////////// SIGNER ///////////////////////// + +pub const SIGNER_IMAGE_DEFAULT: &str = "ghcr.io/commit-boost/signer:latest"; pub const SIGNER_SERVICE_NAME: &str = "signer"; /// Where the signer module should open the server diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 345dc015..3a3c23d8 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -1,7 +1,6 @@ use std::path::PathBuf; -use docker_image::DockerImage; -use eyre::{Result, bail, ensure}; +use eyre::{Result, bail}; use serde::{Deserialize, Serialize}; use crate::types::{Chain, ChainLoader, ForkVersion, load_chain_from_file}; @@ -37,9 +36,6 @@ pub struct CommitBoostConfig { pub metrics: Option, #[serde(default)] pub logs: LogsSettings, - /// Docker image for Commit-Boost services - #[serde(default = "default_image")] - pub docker_image: String, } impl CommitBoostConfig { @@ -56,13 +52,6 @@ impl CommitBoostConfig { ) } - // The Docker tag must parse - ensure!(!self.docker_image.is_empty(), "Docker image is empty"); - ensure!( - DockerImage::parse(&self.docker_image).is_ok(), - format!("Invalid Docker image: {}", self.docker_image) - ); - Ok(()) } @@ -118,7 +107,6 @@ impl CommitBoostConfig { signer: helper_config.signer, metrics: helper_config.metrics, logs: helper_config.logs, - docker_image: helper_config.docker_image, }; Ok(config) @@ -158,9 +146,4 @@ struct HelperConfig { metrics: Option, #[serde(default)] logs: LogsSettings, - docker_image: String, -} - -fn default_image() -> String { - CB_IMAGE_DEFAULT.to_string() } diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index abcc9726..a6e9d908 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -10,6 +10,7 @@ use alloy::{ primitives::{U256, utils::format_ether}, providers::{Provider, ProviderBuilder}, }; +use docker_image::DockerImage; use eyre::{Result, ensure}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; @@ -21,8 +22,8 @@ use super::{ use crate::{ commit::client::SignerClient, config::{ - CB_MODULE_NAME, CONFIG_ENV, MODULE_JWT_ENV, MuxKeysLoader, PbsMuxes, SIGNER_URL_ENV, - load_env_var, load_file_from_env, + CONFIG_ENV, MODULE_JWT_ENV, MuxKeysLoader, PBS_IMAGE_DEFAULT, PBS_SERVICE_NAME, PbsMuxes, + SIGNER_URL_ENV, load_env_var, load_file_from_env, }, pbs::{ DEFAULT_PBS_PORT, DEFAULT_REGISTRY_REFRESH_SECONDS, DefaultTimeout, LATE_IN_SLOT_TIME_MS, @@ -203,6 +204,9 @@ impl PbsConfig { /// Static pbs config from config file #[derive(Debug, Deserialize, Serialize)] pub struct StaticPbsConfig { + /// Docker image of the module + #[serde(default = "default_pbs")] + pub docker_image: String, /// Config of pbs module #[serde(flatten)] pub pbs_config: PbsConfig, @@ -212,11 +216,17 @@ pub struct StaticPbsConfig { } impl StaticPbsConfig { - /// Validate the static pbs config + /// Validate static pbs config pub async fn validate(&self, chain: Chain) -> Result<()> { - self.pbs_config.validate(chain).await?; + // The Docker tag must parse + ensure!(!self.docker_image.is_empty(), "Docker image is empty"); + ensure!( + DockerImage::parse(&self.docker_image).is_ok(), + format!("Invalid Docker image: {}", self.docker_image) + ); - Ok(()) + // Validate the inner pbs config + self.pbs_config.validate(chain).await } } @@ -243,6 +253,10 @@ pub struct PbsModuleConfig { pub mux_lookup: Option>, } +fn default_pbs() -> String { + PBS_IMAGE_DEFAULT.to_string() +} + /// Loads the default pbs config, i.e. with no signer client or custom data pub async fn load_pbs_config() -> Result { let config = CommitBoostConfig::from_env_path()?; @@ -383,7 +397,7 @@ pub async fn load_pbs_custom_config() -> Result<(PbsModuleC Some(SignerClient::new( signer_server_url, module_jwt, - ModuleId(CB_MODULE_NAME.to_string()), + ModuleId(PBS_SERVICE_NAME.to_string()), )?) } else { None diff --git a/crates/common/src/config/signer.rs b/crates/common/src/config/signer.rs index c5f70ac0..dc0df320 100644 --- a/crates/common/src/config/signer.rs +++ b/crates/common/src/config/signer.rs @@ -4,6 +4,7 @@ use std::{ path::PathBuf, }; +use docker_image::DockerImage; use eyre::{OptionExt, Result, bail, ensure}; use serde::{Deserialize, Serialize}; use tonic::transport::{Certificate, Identity}; @@ -16,7 +17,9 @@ use super::{ load_optional_env_var, utils::load_env_var, }; use crate::{ - config::{DIRK_CA_CERT_ENV, DIRK_CERT_ENV, DIRK_DIR_SECRETS_ENV, DIRK_KEY_ENV}, + config::{ + DIRK_CA_CERT_ENV, DIRK_CERT_ENV, DIRK_DIR_SECRETS_ENV, DIRK_KEY_ENV, SIGNER_IMAGE_DEFAULT, + }, signer::{ProxyStore, SignerLoader}, types::{Chain, ModuleId}, utils::{default_host, default_u16, default_u32}, @@ -32,6 +35,10 @@ pub struct SignerConfig { #[serde(default = "default_u16::")] pub port: u16, + /// Docker image of the module + #[serde(default = "default_signer_image")] + pub docker_image: String, + /// Number of JWT auth failures before rate limiting an endpoint /// If set to 0, no rate limiting will be applied #[serde(default = "default_u32::")] @@ -53,10 +60,21 @@ impl SignerConfig { // Port must be positive ensure!(self.port > 0, "Port must be positive"); + // The Docker tag must parse + ensure!(!self.docker_image.is_empty(), "Docker image is empty"); + ensure!( + DockerImage::parse(&self.docker_image).is_ok(), + format!("Invalid Docker image: {}", self.docker_image) + ); + Ok(()) } } +fn default_signer_image() -> String { + SIGNER_IMAGE_DEFAULT.to_string() +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "snake_case")] pub struct DirkHostConfig { diff --git a/justfile b/justfile index a3fcd2e0..4b2e1d11 100644 --- a/justfile +++ b/justfile @@ -29,10 +29,18 @@ build-cli-multiarch version: \ (_docker-build-binary-multiarch version "cli") [doc(""" - Builds the commit-boost-pbs binary to './build/'. + Builds the commit-boost binary to './build/'. """)] -build-pbs-bin version: \ - (_docker-build-binary version "pbs") +build-cb-bin version: \ + (_docker-build-binary version "commit-boost") + +[doc(""" + Builds amd64 and arm64 binaries for the commit-boost crate to './build//', where '' is the + OS / arch platform of the binary (linux_amd64 and linux_arm64). + Used when creating the pbs Docker image. +""")] +build-cb-bin-multiarch version: \ + (_docker-build-binary-multiarch version "commit-boost") [doc(""" Creates a Docker image named 'commit-boost/pbs:' and loads it to the local Docker repository. @@ -43,20 +51,12 @@ build-pbs-img version: \ (_docker-build-image version "pbs") [doc(""" - Builds the commit-boost-pbs binary to './build/' and creates a Docker image named 'commit-boost/pbs:'. + Builds the commit-boost binary to './build/' and creates a Docker image named 'commit-boost/pbs:'. """)] build-pbs version: \ - (build-pbs-bin version) \ + (build-cb-bin version) \ (build-pbs-img version) -[doc(""" - Builds amd64 and arm64 binaries for the commit-boost-pbs crate to './build//', where '' is the - OS / arch platform of the binary (linux_amd64 and linux_arm64). - Used when creating the pbs Docker image. -""")] -build-pbs-bin-multiarch version: \ - (_docker-build-binary-multiarch version "pbs") - [doc(""" Creates a multiarch Docker image manifest named 'commit-boost/pbs:' and pushes it to a custom Docker registry (such as '192.168.1.10:5000'). @@ -66,22 +66,16 @@ build-pbs-img-multiarch version local-docker-registry: \ (_docker-build-image-multiarch version "pbs" local-docker-registry) [doc(""" - Builds amd64 and arm64 binaries for the commit-boost-pbs crate to './build//', where '' is the + Builds amd64 and arm64 binaries for the commit-boost crate to './build//', where '' is the OS / arch platform of the binary (linux_amd64 and linux_arm64). Creates a multiarch Docker image manifest named 'commit-boost/pbs:' and pushes it to a custom Docker registry (such as '192.168.1.10:5000'). Used for testing multiarch images locally instead of using a public registry like GHCR or Docker Hub. """)] build-pbs-multiarch version local-docker-registry: \ - (build-pbs-bin-multiarch version) \ + (build-cb-bin-multiarch version) \ (build-pbs-img-multiarch version local-docker-registry) -[doc(""" - Builds the commit-boost-signer binary to './build/'. -""")] -build-signer-bin version: \ - (_docker-build-binary version "signer") - [doc(""" Creates a Docker image named 'commit-boost/signer:' and loads it to the local Docker repository. Requires the binary to be built first, but this command won't build it automatically if you just need to build the @@ -91,20 +85,12 @@ build-signer-img version: \ (_docker-build-image version "signer") [doc(""" - Builds the commit-boost-signer binary to './build/' and creates a Docker image named 'commit-boost/signer:'. + Builds the commit-boost binary to './build/' and creates a Docker image named 'commit-boost/signer:'. """)] build-signer version: \ - (build-signer-bin version) \ + (build-cb-bin version) \ (build-signer-img version) -[doc(""" - Builds amd64 and arm64 binaries for the commit-boost-signer crate to './build//', where '' is - the OS / arch platform of the binary (linux_amd64 and linux_arm64). - Used when creating the signer Docker image. -""")] -build-signer-bin-multiarch version: \ - (_docker-build-binary-multiarch version "signer") - [doc(""" Creates a multiarch Docker image manifest named 'commit-boost/signer:' and pushes it to a custom Docker registry (such as '192.168.1.10:5000'). @@ -114,14 +100,14 @@ build-signer-img-multiarch version local-docker-registry: \ (_docker-build-image-multiarch version "signer" local-docker-registry) [doc(""" - Builds amd64 and arm64 binaries for the commit-boost-signer crate to './build//', where '' is + Builds amd64 and arm64 binaries for the commit-boost crate to './build//', where '' is the OS / arch platform of the binary (linux_amd64 and linux_arm64). Creates a multiarch Docker image manifest named 'commit-boost/signer:' and pushes it to a custom Docker registry (such as '192.168.1.10:5000'). Used for testing multiarch images locally instead of using a public registry like GHCR or Docker Hub. """)] build-signer-multiarch version local-docker-registry: \ - (build-signer-bin-multiarch version) \ + (build-cb-bin-multiarch version) \ (build-signer-img-multiarch version local-docker-registry) [doc(""" @@ -132,8 +118,9 @@ build-signer-multiarch version local-docker-registry: \ """)] build-all version: \ (build-cli version) \ - (build-pbs version) \ - (build-signer version) + (build-cb-bin version) \ + (build-pbs-img version) \ + (build-signer-img version) [doc(""" Builds amd64 and arm64 flavors of the CLI, PBS, and Signer binaries and Docker images for the specified version. @@ -145,8 +132,9 @@ build-all version: \ """)] build-all-multiarch version local-docker-registry: \ (build-cli-multiarch version) \ - (build-pbs-multiarch version local-docker-registry) \ - (build-signer-multiarch version local-docker-registry) + (build-cb-bin-multiarch version) \ + (build-pbs-img-multiarch version local-docker-registry) \ + (build-signer-img-multiarch version local-docker-registry) # =============================== # === Builder Implementations === diff --git a/provisioning/pbs.Dockerfile b/provisioning/pbs.Dockerfile index 6b9496ec..6a4c4646 100644 --- a/provisioning/pbs.Dockerfile +++ b/provisioning/pbs.Dockerfile @@ -1,6 +1,6 @@ FROM debian:bookworm-slim ARG BINARIES_PATH TARGETOS TARGETARCH -COPY ${BINARIES_PATH}/${TARGETOS}_${TARGETARCH}/commit-boost-pbs /usr/local/bin/commit-boost-pbs +COPY ${BINARIES_PATH}/${TARGETOS}_${TARGETARCH}/commit-boost /usr/local/bin/commit-boost RUN apt-get update && apt-get install -y \ openssl \ ca-certificates \ @@ -16,4 +16,5 @@ RUN groupadd -g 10001 commitboost && \ useradd -u 10001 -g commitboost -s /sbin/nologin commitboost USER commitboost -ENTRYPOINT ["/usr/local/bin/commit-boost-pbs"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/commit-boost"] +CMD ["pbs"] \ No newline at end of file diff --git a/provisioning/signer.Dockerfile b/provisioning/signer.Dockerfile index 5ea619b2..d6c3d498 100644 --- a/provisioning/signer.Dockerfile +++ b/provisioning/signer.Dockerfile @@ -1,6 +1,6 @@ FROM debian:bookworm-slim ARG BINARIES_PATH TARGETOS TARGETARCH -COPY ${BINARIES_PATH}/${TARGETOS}_${TARGETARCH}/commit-boost-signer /usr/local/bin/commit-boost-signer +COPY ${BINARIES_PATH}/${TARGETOS}_${TARGETARCH}/commit-boost /usr/local/bin/commit-boost RUN apt-get update && apt-get install -y \ openssl \ ca-certificates \ @@ -16,4 +16,5 @@ RUN groupadd -g 10001 commitboost && \ useradd -u 10001 -g commitboost -s /sbin/nologin commitboost USER commitboost -ENTRYPOINT ["/usr/local/bin/commit-boost-signer"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/commit-boost"] +CMD ["signer"] \ No newline at end of file diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 826ad769..58ef42cf 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -7,9 +7,9 @@ use std::{ use alloy::primitives::U256; use cb_common::{ config::{ - PbsConfig, PbsModuleConfig, RelayConfig, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, - SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, - SignerType, StartSignerConfig, + PbsConfig, PbsModuleConfig, RelayConfig, SIGNER_IMAGE_DEFAULT, + SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, + SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, }, pbs::{RelayClient, RelayEntry}, signer::SignerLoader, @@ -109,6 +109,7 @@ pub fn get_signer_config(loader: SignerLoader) -> SignerConfig { SignerConfig { host: default_host(), port: SIGNER_PORT_DEFAULT, + docker_image: SIGNER_IMAGE_DEFAULT.to_string(), jwt_auth_fail_limit: SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, jwt_auth_fail_timeout_seconds: SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, inner: SignerType::Local { loader, store: None }, diff --git a/tests/tests/config.rs b/tests/tests/config.rs index 1f4f6b4d..bffefcbc 100644 --- a/tests/tests/config.rs +++ b/tests/tests/config.rs @@ -36,7 +36,7 @@ async fn test_load_pbs_happy() -> Result<()> { ); // Docker and general settings - assert_eq!(config.docker_image, "ghcr.io/commit-boost/commit-boost:latest"); + assert_eq!(config.pbs.docker_image, "ghcr.io/commit-boost/pbs:latest"); assert!(!config.pbs.with_signer); assert_eq!(config.pbs.pbs_config.host, "127.0.0.1".parse::().unwrap()); assert_eq!(config.pbs.pbs_config.port, 18550); From 7dec4d35697bfff7c8412ff9a9881759e3d819e3 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 3 Feb 2026 08:59:56 -0500 Subject: [PATCH 06/22] Updated justfile --- bin/cli.rs | 24 ------------------------ bin/commit-boost.rs | 2 +- justfile | 35 ++++++++++------------------------- 3 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 bin/cli.rs diff --git a/bin/cli.rs b/bin/cli.rs deleted file mode 100644 index 0f36d388..00000000 --- a/bin/cli.rs +++ /dev/null @@ -1,24 +0,0 @@ -use clap::Parser; - -/// Version string with a leading 'v' -const VERSION: &str = concat!("v", env!("CARGO_PKG_VERSION")); - -/// Subcommands and global arguments for the module -#[derive(Parser, Debug)] -#[command(name = "Commit-Boost CLI", version = VERSION, about, long_about = None)] -struct Cli {} - -/// Main entry point of the Commit-Boost CLI -#[tokio::main] -async fn main() -> eyre::Result<()> { - // Parse the CLI arguments (currently only used for version info, more can be - // added later) - let _cli = Cli::parse(); - - color_eyre::install()?; - // set default backtrace unless provided - - let args = cb_cli::CliArgs::parse(); - - args.run().await -} diff --git a/bin/commit-boost.rs b/bin/commit-boost.rs index 89f1a27e..7976831c 100644 --- a/bin/commit-boost.rs +++ b/bin/commit-boost.rs @@ -35,7 +35,7 @@ enum Commands { /// Run the Signer service Signer, - /// Generate the starting docker-compose file (the old CLI binary) + /// Generate the starting docker-compose files and environment files Init { /// Path to config file #[arg(long("config"))] diff --git a/justfile b/justfile index 4b2e1d11..b39ab9f4 100644 --- a/justfile +++ b/justfile @@ -15,23 +15,10 @@ clippy: # === Build Commands for Services === # =================================== -[doc(""" - Builds the commit-boost-cli binary to './build/'. -""")] -build-cli version: \ - (_docker-build-binary version "cli") - -[doc(""" - Builds amd64 and arm64 binaries for the commit-boost-cli crate to './build//', where '' is - the OS / arch platform of the binary (linux_amd64 and linux_arm64). -""")] -build-cli-multiarch version: \ - (_docker-build-binary-multiarch version "cli") - [doc(""" Builds the commit-boost binary to './build/'. """)] -build-cb-bin version: \ +build-bin version: \ (_docker-build-binary version "commit-boost") [doc(""" @@ -39,7 +26,7 @@ build-cb-bin version: \ OS / arch platform of the binary (linux_amd64 and linux_arm64). Used when creating the pbs Docker image. """)] -build-cb-bin-multiarch version: \ +build-bin-multiarch version: \ (_docker-build-binary-multiarch version "commit-boost") [doc(""" @@ -54,7 +41,7 @@ build-pbs-img version: \ Builds the commit-boost binary to './build/' and creates a Docker image named 'commit-boost/pbs:'. """)] build-pbs version: \ - (build-cb-bin version) \ + (build-bin version) \ (build-pbs-img version) [doc(""" @@ -73,7 +60,7 @@ build-pbs-img-multiarch version local-docker-registry: \ Used for testing multiarch images locally instead of using a public registry like GHCR or Docker Hub. """)] build-pbs-multiarch version local-docker-registry: \ - (build-cb-bin-multiarch version) \ + (build-bin-multiarch version) \ (build-pbs-img-multiarch version local-docker-registry) [doc(""" @@ -88,7 +75,7 @@ build-signer-img version: \ Builds the commit-boost binary to './build/' and creates a Docker image named 'commit-boost/signer:'. """)] build-signer version: \ - (build-cb-bin version) \ + (build-bin version) \ (build-signer-img version) [doc(""" @@ -107,7 +94,7 @@ build-signer-img-multiarch version local-docker-registry: \ Used for testing multiarch images locally instead of using a public registry like GHCR or Docker Hub. """)] build-signer-multiarch version local-docker-registry: \ - (build-cb-bin-multiarch version) \ + (build-bin-multiarch version) \ (build-signer-img-multiarch version local-docker-registry) [doc(""" @@ -117,8 +104,7 @@ build-signer-multiarch version local-docker-registry: \ 'commit-boost/signer:'. """)] build-all version: \ - (build-cli version) \ - (build-cb-bin version) \ + (build-bin version) \ (build-pbs-img version) \ (build-signer-img version) @@ -131,8 +117,7 @@ build-all version: \ Used for testing multiarch images locally instead of using a public registry like GHCR or Docker Hub. """)] build-all-multiarch version local-docker-registry: \ - (build-cli-multiarch version) \ - (build-cb-bin-multiarch version) \ + (build-bin-multiarch version) \ (build-pbs-img-multiarch version local-docker-registry) \ (build-signer-img-multiarch version local-docker-registry) @@ -147,7 +132,7 @@ _create-docker-builder: # Builds a binary for a specific crate and version _docker-build-binary version crate: _create-docker-builder export PLATFORM=$(docker buildx inspect --bootstrap | awk -F': ' '/Platforms/ {print $2}' | cut -d',' -f1 | xargs | tr '/' '_'); \ - docker buildx build --rm --platform=local -f provisioning/build.Dockerfile --output "build/{{version}}/$PLATFORM" --target output --build-arg TARGET_CRATE=commit-boost-{{crate}} . + docker buildx build --rm --platform=local -f provisioning/build.Dockerfile --output "build/{{version}}/$PLATFORM" --target output --build-arg TARGET_CRATE=commit-boost . # Builds a Docker image for a specific crate and version _docker-build-image version crate: _create-docker-builder @@ -155,7 +140,7 @@ _docker-build-image version crate: _create-docker-builder # Builds multiple binaries (for Linux amd64 and arm64 architectures) for a specific crate and version _docker-build-binary-multiarch version crate: _create-docker-builder - docker buildx build --rm --platform=linux/amd64,linux/arm64 -f provisioning/build.Dockerfile --output build/{{version}} --target output --build-arg TARGET_CRATE=commit-boost-{{crate}} . + docker buildx build --rm --platform=linux/amd64,linux/arm64 -f provisioning/build.Dockerfile --output build/{{version}} --target output --build-arg TARGET_CRATE=commit-boost . # Builds a multi-architecture (Linux amd64 and arm64) Docker manifest for a specific crate and version. # Uploads to the custom Docker registry (such as '192.168.1.10:5000') instead of a public registry like GHCR or Docker Hub. From 206658b152870413d17f573c0100bd73ae286e9b Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 3 Feb 2026 09:49:39 -0500 Subject: [PATCH 07/22] Updated release workflow --- .github/workflows/release.yml | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69725a28..09ba879d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ permissions: packages: write jobs: - # Builds the x64 and arm64 binaries for Linux, for all 3 crates, via the Docker builder + # Builds the x64 and arm64 binary for Linux via the Docker builder build-binaries-linux: strategy: matrix: @@ -18,15 +18,12 @@ jobs: - amd64 - arm64 name: - - commit-boost-cli - commit-boost include: - target: amd64 package-suffix: x86-64 - target: arm64 package-suffix: arm64 - - name: commit-boost-cli - target-crate: cli - name: commit-boost target-crate: commit-boost runs-on: ubuntu-latest @@ -80,7 +77,7 @@ jobs: path: | ${{ matrix.name }}-${{ github.ref_name }}-linux_${{ matrix.package-suffix }}.tar.gz - # Builds the arm64 binaries for Darwin, for all 3 crates, natively + # Builds the arm64 binary for Darwin natively build-binaries-darwin: strategy: matrix: @@ -89,9 +86,7 @@ jobs: # - x86_64-apple-darwin - aarch64-apple-darwin name: - - commit-boost-cli - - commit-boost-pbs - - commit-boost-signer + - commit-boost include: # - target: x86_64-apple-darwin # os: macos-latest-large @@ -170,16 +165,16 @@ jobs: uses: actions/download-artifact@v4 with: path: ./artifacts - pattern: "commit-boost-*" + pattern: "commit-boost*" - name: Extract binaries run: | mkdir -p ./artifacts/bin/linux_amd64 mkdir -p ./artifacts/bin/linux_arm64 - tar -xzf ./artifacts/commit-boost-pbs-${{ github.ref_name }}-linux_x86-64/commit-boost-pbs-${{ github.ref_name }}-linux_x86-64.tar.gz -C ./artifacts/bin - mv ./artifacts/bin/commit-boost-pbs ./artifacts/bin/linux_amd64/commit-boost-pbs - tar -xzf ./artifacts/commit-boost-pbs-${{ github.ref_name }}-linux_arm64/commit-boost-pbs-${{ github.ref_name }}-linux_arm64.tar.gz -C ./artifacts/bin - mv ./artifacts/bin/commit-boost-pbs ./artifacts/bin/linux_arm64/commit-boost-pbs + tar -xzf ./artifacts/commit-boost-${{ github.ref_name }}-linux_x86-64/commit-boost-${{ github.ref_name }}-linux_x86-64.tar.gz -C ./artifacts/bin + mv ./artifacts/bin/commit-boost ./artifacts/bin/linux_amd64/commit-boost + tar -xzf ./artifacts/commit-boost-${{ github.ref_name }}-linux_arm64/commit-boost-${{ github.ref_name }}-linux_arm64.tar.gz -C ./artifacts/bin + mv ./artifacts/bin/commit-boost ./artifacts/bin/linux_arm64/commit-boost - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -222,16 +217,16 @@ jobs: uses: actions/download-artifact@v4 with: path: ./artifacts - pattern: "commit-boost-*" + pattern: "commit-boost*" - name: Extract binaries run: | mkdir -p ./artifacts/bin/linux_amd64 mkdir -p ./artifacts/bin/linux_arm64 - tar -xzf ./artifacts/commit-boost-signer-${{ github.ref_name }}-linux_x86-64/commit-boost-signer-${{ github.ref_name }}-linux_x86-64.tar.gz -C ./artifacts/bin - mv ./artifacts/bin/commit-boost-signer ./artifacts/bin/linux_amd64/commit-boost-signer - tar -xzf ./artifacts/commit-boost-signer-${{ github.ref_name }}-linux_arm64/commit-boost-signer-${{ github.ref_name }}-linux_arm64.tar.gz -C ./artifacts/bin - mv ./artifacts/bin/commit-boost-signer ./artifacts/bin/linux_arm64/commit-boost-signer + tar -xzf ./artifacts/commit-boost-${{ github.ref_name }}-linux_x86-64/commit-boost-${{ github.ref_name }}-linux_x86-64.tar.gz -C ./artifacts/bin + mv ./artifacts/bin/commit-boost ./artifacts/bin/linux_amd64/commit-boost + tar -xzf ./artifacts/commit-boost-${{ github.ref_name }}-linux_arm64/commit-boost-${{ github.ref_name }}-linux_arm64.tar.gz -C ./artifacts/bin + mv ./artifacts/bin/commit-boost ./artifacts/bin/linux_arm64/commit-boost - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -272,7 +267,7 @@ jobs: uses: actions/download-artifact@v4 with: path: ./artifacts - pattern: "commit-boost-*" + pattern: "commit-boost*" - name: Finalize Release uses: softprops/action-gh-release@v2 From c2ff756ba78cb95a12a956829752bf4c0572a79f Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 3 Feb 2026 11:58:55 -0500 Subject: [PATCH 08/22] Updated docs for unified binary --- docs/docs/get_started/building.md | 46 +++++++++---------------- docs/docs/get_started/overview.md | 37 +++++++------------- docs/docs/get_started/running/docker.md | 24 ++++++------- 3 files changed, 42 insertions(+), 65 deletions(-) diff --git a/docs/docs/get_started/building.md b/docs/docs/get_started/building.md index 3a0a964a..81968dbc 100644 --- a/docs/docs/get_started/building.md +++ b/docs/docs/get_started/building.md @@ -14,15 +14,15 @@ The build system assumes that you've added your user account to the `docker` gro The Docker builder is built into the project's `justfile` which is used to invoke many facets of Commit Boost development. To use it, you'll need to install [Just](https://github.com/casey/just) on your system. -Use `just --list` to show all of the actions - there are many. The `justfile` provides granular actions, called "recipes", for building just the binaries of a specific crate (such as the CLI, `pbs`, or `signer`), as well as actions to build the Docker images for the PBS and Signer modules. +Use `just --list` to show all of the actions - there are many. The `justfile` provides granular actions, called "recipes", for building just the binaries of a specific crate (such as the CLI, `pbs`, or `signer`), as well as actions to build the Docker images for the PBS and Signer services. Below is a brief summary of the relevant ones for building the Commit-Boost artifacts: -- `build-all ` will build the `commit-boost-cli`, `commit-boost-pbs`, and `commit-boost-signer` binaries for your local system architecture. It will also create Docker images called `commit-boost/pbs:` and `commit-boost/signer:` and load them into your local Docker registry for use. -- `build-cli-bin `, `build-pbs-bin `, and `build-signer-bin ` can be used to create the `commit-boost-cli`, `commit-boost-pbs`, and `commit-boost-signer` binaries, respectively. -- `build-pbs-img ` and `build-signer-img ` can be used to create the Docker images for the PBS and Signer modules, respectively. +- `build-all ` will build the `commit-boost` binary for your local system architecture. It will also create Docker images called `commit-boost/pbs:` and `commit-boost/signer:` and load them into your local Docker registry for use. +- `build-bin ` can be used to create the `commit-boost` binary itself. +- `build-pbs-img ` and `build-signer-img ` can be used to create the Docker images for the PBS and Signer services, respectively. -The `version` provided will be used to house the output binaries in `./build/`, and act as the version tag for the Docker images when they're added to your local system or uploaded to your local Docker repository. +The `version` provided will be used to house the output binaries in `./build/`, and act as the version tag for the Docker images when they're added to your local system or uploaded to your local Docker repository. For example, using `$(git rev-parse --short HEAD)` will set the version to the current commit hash. If you're interested in building the binaries and/or Docker images for multiple architectures (currently Linux `amd64` and `arm64`), use the variants of those recipes that have the `-multiarch` suffix. Note that building a multiarch Docker image manifest will require the use of a [custom Docker registry](https://www.digitalocean.com/community/tutorials/how-to-set-up-a-private-docker-registry-on-ubuntu-20-04), as the local registry built into Docker does not have multiarch manifest support. @@ -81,31 +81,25 @@ git submodule update --init --recursive Your build environment should now be ready to use. -### Building the CLI +### Building the Binary -To build the CLI, run: +To build the binary, run: ``` -cargo build --release --bin commit-boost-cli +just build-bin ``` -This will create a binary in `./target/release/commit-boost-cli`. Confirm that it works: +This will create a binary in `build//`, for example `build/206658b/linux_amd64/`. Confirm that it works: ``` -./target/release/commit-boost-cli --version +./build///commit-boost --version ``` You can now use this to generate the Docker Compose file to drive the other modules if desired. See the [configuration](./configuration.md) guide for more information. -### Building the PBS Module +### Verifying the PBS Service -To build PBS, run: - -``` -cargo build --release --bin commit-boost-pbs -``` - -This will create a binary in `./target/release/commit-boost-pbs`. To verify it works, create [a TOML configuration](./configuration.md) for the PBS module (e.g., `cb-config.toml`). +To verify the PBS service works, create [a TOML configuration](./configuration.md) for the PBS module (e.g., `cb-config.toml`). As a quick example, we'll use this configuration that connects to the Flashbots relay on the Hoodi network: @@ -134,7 +128,7 @@ secrets_path = "/tmp/secrets" Set the path to it in the `CB_CONFIG` environment variable and run the binary: ``` -CB_CONFIG=cb-config.toml ./target/release/commit-boost-pbs +CB_CONFIG=cb-config.toml ./build///commit-boost pbs ``` If it works, you should see output like this: @@ -146,17 +140,11 @@ If it works, you should see output like this: 2025-05-07T21:09:17.896196Z INFO : relay check successful method=/eth/v1/builder/status req_id=5c405c33-0496-42ea-a35d-a7a01dbba356 ``` -If you do, then the binary works. +If you do, then the PBS service works. -### Building the Signer Module - -To build the Signer, run: - -``` -cargo build --release --bin commit-boost-signer -``` +### Verifying the Signer Module -This will create a binary in `./target/release/commit-boost-signer`. To verify it works, create [a TOML configuration](./configuration.md) for the Signer module (e.g., `cb-config.toml`). We'll use the example in the PBS build section above. +To verify the Signer service works, create [a TOML configuration](./configuration.md) for the Signer module (e.g., `cb-config.toml`). We'll use the example in the PBS section above. The signer needs the following environment variables set: @@ -167,7 +155,7 @@ Set these values, create the `keys` and `secrets` directories listed in the conf ``` mkdir -p /tmp/keys && mkdir -p /tmp/secrets -CB_CONFIG=cb-config.toml CB_JWTS="test_jwts=dummy" ./target/release/commit-boost-signer +CB_CONFIG=cb-config.toml CB_JWTS="test_jwts=dummy" ./build///commit-boost signer ``` You should see output like this: diff --git a/docs/docs/get_started/overview.md b/docs/docs/get_started/overview.md index e5209f09..fb4f8a13 100644 --- a/docs/docs/get_started/overview.md +++ b/docs/docs/get_started/overview.md @@ -7,14 +7,14 @@ description: Initial setup Commit-Boost is primarily based on [Docker](https://www.docker.com/) to enable modularity, sandboxing and cross-platform compatibility. It is also possible to run Commit-Boost [natively](/get_started/running/binary) without Docker. Each component roughly maps to a container: from a single `.toml` config file, the node operator can specify which modules they want to run, and Commit-Boost takes care of spinning up the services and creating links between them. -Commit-Boost ships with two core modules: +Commit-Boost ships with two core services: - A PBS module which implements the [BuilderAPI](https://ethereum.github.io/builder-specs/) for [MEV Boost](https://docs.flashbots.net/flashbots-mev-boost/architecture-overview/specifications). - A signer module, which implements the [Signer API](/api) and provides the interface for modules to request proposer commitments. ## Setup -The Commit-Boost CLI creates a dynamic `docker-compose` file, with services and ports already set up. +The Commit-Boost program can create a dynamic `docker-compose` file, with services and ports already set up. Whether you're using Docker or running the binaries natively, you can compile from source directly from the repo, or download binaries and fetch docker images from the official releases. @@ -22,7 +22,7 @@ Whether you're using Docker or running the binaries natively, you can compile fr Find the latest releases at https://github.com/Commit-Boost/commit-boost-client/releases. -The modules are also published at [each release](https://github.com/orgs/Commit-Boost/packages?repo_name=commit-boost-client). +The services are also published at [each release](https://github.com/orgs/Commit-Boost/packages?repo_name=commit-boost-client). ### From source @@ -49,35 +49,24 @@ git submodule update --init --recursive If you get an `openssl` related error try running: `apt-get update && apt-get install -y openssl ca-certificates libssl3 libssl-dev build-essential pkg-config` ::: -### Docker - -You will need to build the CLI to create the `docker-compose` file: +Now, build the binary, which will be stored in `build//`, for example `build/206658b/linux_amd64/`: ```bash -# Build the CLI -cargo build --release --bin commit-boost-cli - -# Check that it works -./target/release/commit-boost-cli --version +just build-bin $(git rev-parse --short HEAD) ``` -and the modules as Docker images: - +You can confirm the binary was built successfully by navigating to the build directory and checking its version: ```bash -docker build -t commitboost_pbs_default . -f ./provisioning/pbs.Dockerfile -docker build -t commitboost_signer . -f ./provisioning/signer.Dockerfile +./commit-boost --version ``` -This will create two local images called `commitboost_pbs_default` and `commitboost_signer` for the Pbs and Signer module respectively. Make sure to use these images in the `docker_image` field in the `[pbs]` and `[signer]` sections of the `.toml` config file, respectively. - -### Binaries +### Docker -Alternatively, you can also build the modules from source and run them without Docker, in which case you can skip the CLI and only compile the modules: +Building the service images requires the binary to be built using the above instructions first, since it will be copied into those images. Once it's built, create the images with the following: ```bash -# Build the PBS module -cargo build --release --bin commit-boost-pbs - -# Build the Signer module -cargo build --release --bin commit-boost-signer +just build-pbs-img $(git rev-parse --short HEAD) +just build-signer-img $(git rev-parse --short HEAD) ``` + +This will create two local images called `commit_boost/pbs:` and `commit_boost/signer:` for the PBS and Signer services respectively. Make sure to use these images in the `docker_image` field in the `[pbs]` and `[signer]` sections of the `.toml` config file, respectively. diff --git a/docs/docs/get_started/running/docker.md b/docs/docs/get_started/running/docker.md index 396fdede..89465a44 100644 --- a/docs/docs/get_started/running/docker.md +++ b/docs/docs/get_started/running/docker.md @@ -3,13 +3,13 @@ description: Run Commit-Boost with Docker --- # Docker -The Commit-Boost CLI generates a dynamic `docker-compose.yml` file using the provided `.toml` config file. This is the recommended approach as Docker provides sandboxing of the containers from the rest of your system. +The Commit-Boost program generates a dynamic `docker-compose.yml` file using the provided `.toml` config file. This is the recommended approach as Docker provides sandboxing of the containers from the rest of your system. ## Init First run: ```bash -commit-boost-cli init --config cb-config.toml +commit-boost init --config cb-config.toml ``` This will create up to three files: - `cb.docker-compose.yml` which contains the full setup of the Commit-Boost services. @@ -73,9 +73,9 @@ Note that there are many more parameters that Commit-Boost supports, but they ar The relays here are placeholder for the sake of the example; for a list of actual relays, visit [the EthStaker relay list](https://github.com/eth-educators/ethstaker-guides/blob/main/MEV-relay-list.md). -### Commit-Boost CLI Output +### Commit-Boost Init Output -Run `commit-boost-cli init --config cb-config.toml` with the above configuration, the CLI will produce the following Docker Compose file: +Run `commit-boost init --config cb-config.toml` with the above configuration, the program will produce the following Docker Compose file: ``` services: @@ -102,14 +102,14 @@ This will run the PBS service in a container named `cb_pbs`. ### Configuration File Volume -The CLI creates a read-only volume binding for the config file, which the PBS service needs to run. The Docker compose file that it creates with the `init` command, `cb.docker-compose.yml`, will be placed into your current working directory when you run the CLI. The volume source will be specified as a *relative path* to that working directory, so it's ideal if the config file is directly within your working directory (or a subdirectory). If you need to specify an absolute path for the config file, you can adjust the `volumes` entry within the Docker compose file manually after its creation. +The program creates a read-only volume binding for the config file, which the PBS service needs to run. The Docker compose file that it creates with the `init` command, `cb.docker-compose.yml`, will be placed into your current working directory when you run the program. The volume source will be specified as a *relative path* to that working directory, so it's ideal if the config file is directly within your working directory (or a subdirectory). If you need to specify an absolute path for the config file, you can adjust the `volumes` entry within the Docker compose file manually after its creation. Since this is a volume, the PBS service container will reload the file from disk any time it's restarted. That means you can change the file any time after the Docker compose file is created to tweak PBS's parameters, but it also means the config file must stay in the same location; if you move it, the PBS container won't be able to mount it anymore and fail to start unless you manually adjust the volume's source location. ### Networking -The CLI will force the PBS service to bind to `0.0.0.0` within Docker's internal network so other Docker containers can access it, but it will only expose the API port (default `18550`) to `127.0.0.1` on your host machine. That way any processes running on the same machine can access it on that port. If you want to open the port for access across your entire network, not just your local machine, you can add the line: +The program will force the PBS service to bind to `0.0.0.0` within Docker's internal network so other Docker containers can access it, but it will only expose the API port (default `18550`) to `127.0.0.1` on your host machine. That way any processes running on the same machine can access it on that port. If you want to open the port for access across your entire network, not just your local machine, you can add the line: ``` host = "0.0.0.0" @@ -124,7 +124,7 @@ to the `[pbs]` section in the configuration. This will cause the resulting `port though you will need to add an entry to your local machine's firewall software (if applicable) for other machines to access it. -Currently, the CLI will always export the PBS service's API port in one of these two ways. If you don't want to expose it at all, so it can only be accessed by other Docker containers running within Docker's internal network, you will need to manually remove the `ports` entry from the Docker compose file after it's been created: +Currently, the program will always export the PBS service's API port in one of these two ways. If you don't want to expose it at all, so it can only be accessed by other Docker containers running within Docker's internal network, you will need to manually remove the `ports` entry from the Docker compose file after it's been created: ``` ports: [] @@ -177,9 +177,9 @@ The relays here are placeholder for the sake of the example; for a list of actua In this scenario there are two folders in the same directory as the configuration file (the working directory): `keys` and `secrets`. These correspond to the folders containing the [EIP-2335 keystores](../configuration.md#local-signer) and secrets in Lighthouse format. For your own keys, adjust the `format` parameter within the configuration and directory paths accordingly. -### Commit-Boost CLI Output +### Commit-Boost Init Output -Run `commit-boost-cli init --config cb-config.toml` with the above configuration, the CLI will produce two files: +Run `commit-boost init --config cb-config.toml` with the above configuration, the program will produce two files: - `cb.docker-compose.yml` - `.cb.env` @@ -261,7 +261,7 @@ CB_JWT_DA_COMMIT=mwDSSr7chwy9eFf7RhedBoyBtrwFUjSQ CB_JWTS=DA_COMMIT=mwDSSr7chwy9eFf7RhedBoyBtrwFUjSQ ``` -The Signer service needs JWT authentication from each of its modules. The CLI creates these and embeds them into the containers via environment variables automatically for convenience. This is demonstrated for the Signer module within the `environment` compose block: the `CB_JWTS: ${CB_JWTS}` forwards the `CB_JWTS` environment variable that's present when running Docker compose. The CLI requests that you do so via the command `docker compose --env-file "./.cb.env" -f "./cb.docker-compose.yml" up -d`; the `--env-file "./.cb.env"` handles loading the CLI's JWT output into this environment variable. +The Signer service needs JWT authentication from each of its modules. The program creates these and embeds them into the containers via environment variables automatically for convenience. This is demonstrated for the Signer module within the `environment` compose block: the `CB_JWTS: ${CB_JWTS}` forwards the `CB_JWTS` environment variable that's present when running Docker compose. The program requests that you do so via the command `docker compose --env-file "./.cb.env" -f "./cb.docker-compose.yml" up -d`; the `--env-file "./.cb.env"` handles loading the program's JWT output into this environment variable. Similarly, for the `cb_da_commit` module, the `CB_SIGNER_JWT: ${CB_JWT_DA_COMMIT}` line within its `environment` block will set the JWT that it should use to authenticate with the Signer service. @@ -273,7 +273,7 @@ As with the PBS-only example, the configuration file is placed into a read-only ### Networking -The CLI will force both the PBS and Signer API endpoints to bind to `0.0.0.0` within Docker's internal network so other Docker containers can access them, but it will only expose the API port (default `18550` for PBS and `20000` for the Signer) to `127.0.0.1` on your host machine. That way any processes running on the same machine can access them on their respective ports. If you want to open the ports for access across your entire network, not just your local machine, you can add the line: +The program will force both the PBS and Signer API endpoints to bind to `0.0.0.0` within Docker's internal network so other Docker containers can access them, but it will only expose the API port (default `18550` for PBS and `20000` for the Signer) to `127.0.0.1` on your host machine. That way any processes running on the same machine can access them on their respective ports. If you want to open the ports for access across your entire network, not just your local machine, you can add the line: ``` host = "0.0.0.0" @@ -296,7 +296,7 @@ to both the `[pbs]` and `[signer]` sections in the configuration. This will caus though you will need to add entries to your local machine's firewall software (if applicable) for other machines to access them. -Currently, the CLI will always export the PBS and Signer services' API ports in one of these two ways. If you don't want to expose them at all, so they can only be accessed by other Docker containers running within Docker's internal network, you will need to manually remove the `ports` entries from the Docker compose files after they've been created: +Currently, the program will always export the PBS and Signer services' API ports in one of these two ways. If you don't want to expose them at all, so they can only be accessed by other Docker containers running within Docker's internal network, you will need to manually remove the `ports` entries from the Docker compose files after they've been created: ``` ports: [] From a62086fa607401560be137e82fcd3a67e0502e96 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 3 Feb 2026 12:00:56 -0500 Subject: [PATCH 09/22] Clippy fix --- crates/cli/src/docker_init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index ada3241d..450aa7bd 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -111,7 +111,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re // this is ok since the config has already been loaded once let filename = spec.file_name().unwrap().to_str().unwrap(); let chain_spec = ServiceChainSpecInfo { - env: get_env_val(CHAIN_SPEC_ENV, &format!("/{}", filename)), + env: get_env_val(CHAIN_SPEC_ENV, &format!("/{filename}")), volume: Volumes::Simple(format!("{}:/{}:ro", spec.display(), filename)), }; service_config.chain_spec = Some(chain_spec); From 67514fe3be1b7dd4999b51378efa7c48c8abf881 Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Tue, 3 Mar 2026 11:57:56 -0500 Subject: [PATCH 10/22] Added integration tests to ensure the CLI commands work --- Cargo.lock | 85 +++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ bin/Cargo.toml | 4 +++ bin/commit-boost.rs | 7 ++-- bin/src/lib.rs | 3 ++ bin/tests/binary.rs | 36 +++++++++++++++++++ 6 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 bin/tests/binary.rs diff --git a/Cargo.lock b/Cargo.lock index 2e23ceba..a991b18b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1090,6 +1090,21 @@ dependencies = [ "serde", ] +[[package]] +name = "assert_cmd" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -1531,6 +1546,17 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -1884,6 +1910,7 @@ dependencies = [ name = "commit-boost" version = "0.9.3" dependencies = [ + "assert_cmd", "cb-cli", "cb-common", "cb-metrics", @@ -1892,6 +1919,7 @@ dependencies = [ "clap", "color-eyre", "eyre", + "predicates", "tokio", "tracing", "tree_hash", @@ -2424,6 +2452,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.9.0" @@ -3029,6 +3063,15 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -4379,6 +4422,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "notify" version = "8.2.0" @@ -4802,6 +4851,36 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "pretty_reqwest_error" version = "0.1.0" @@ -6363,6 +6442,12 @@ dependencies = [ "windows-sys 0.61.0", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "test_random_derive" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 09b70e30..e2fcf44f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ alloy = { version = "^1.0.35", features = [ "ssz", ] } alloy-primitives = "^1.3.1" +assert_cmd = "2.1.2" async-trait = "0.1.80" axum = { version = "0.8.1", features = ["macros"] } axum-extra = { version = "0.10.0", features = ["typed-header"] } @@ -54,6 +55,7 @@ lh_types = { package = "types", git = "https://github.com/sigp/lighthouse", tag notify = "8.2.0" parking_lot = "0.12.3" pbkdf2 = "0.12.2" +predicates = "3.0.3" prometheus = "0.14.0" prost = "0.13.4" rand = { version = "0.9", features = ["os_rng"] } diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 19744c1f..0a3395a9 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -18,6 +18,10 @@ tracing.workspace = true tree_hash.workspace = true tree_hash_derive.workspace = true +[dev-dependencies] +assert_cmd.workspace = true +predicates.workspace = true + [[bin]] name = "commit-boost" path = "commit-boost.rs" diff --git a/bin/commit-boost.rs b/bin/commit-boost.rs index 4088a272..83ff247a 100644 --- a/bin/commit-boost.rs +++ b/bin/commit-boost.rs @@ -13,15 +13,12 @@ use clap::{Parser, Subcommand}; use eyre::Result; use tracing::{error, info}; -/// Version string with a leading 'v' -const VERSION: &str = concat!("v", env!("CARGO_PKG_VERSION")); - /// Long about string for the CLI const LONG_ABOUT: &str = "Commit-Boost allows Ethereum validators to safely run MEV-Boost and community-built commitment protocols"; /// Subcommands and global arguments for the module #[derive(Parser, Debug)] -#[command(name = "Commit-Boost", version = VERSION, about, long_about = LONG_ABOUT)] +#[command(name = "Commit-Boost", version = commit_boost::VERSION, about, long_about = LONG_ABOUT)] struct Cli { #[command(subcommand)] command: Commands, @@ -115,6 +112,8 @@ async fn run_init(config_path: PathBuf, output_path: PathBuf) -> Result<()> { #[cfg(test)] mod tests { + use commit_boost::VERSION; + use super::*; #[test] diff --git a/bin/src/lib.rs b/bin/src/lib.rs index 4bb668e6..487a46ef 100644 --- a/bin/src/lib.rs +++ b/bin/src/lib.rs @@ -24,3 +24,6 @@ pub mod prelude { } pub use tree_hash_derive::TreeHash; } + +/// Version string with a leading 'v' +pub const VERSION: &str = concat!("v", env!("CARGO_PKG_VERSION")); diff --git a/bin/tests/binary.rs b/bin/tests/binary.rs new file mode 100644 index 00000000..8c524c8f --- /dev/null +++ b/bin/tests/binary.rs @@ -0,0 +1,36 @@ +use assert_cmd::{Command, cargo}; + +/// Tests that the binary can be run and returns a version string +#[test] +fn test_load_example_config() { + let mut cmd = Command::new(cargo::cargo_bin!()); + let expected_version = format!("Commit-Boost {}\n", commit_boost::VERSION); + cmd.arg("--version").assert().success().stdout(expected_version); +} + +/// Tests that the init command can be run and complains about not having +/// --config set +#[test] +fn test_run_init() { + let mut cmd = Command::new(cargo::cargo_bin!()); + cmd.args(["init"]).assert().failure().stderr(predicates::str::contains( + "error: the following required arguments were not provided:\n --config ", + )); +} + +/// Tests that PBS runs without CB_CONFIG being set and complains normally +#[test] +fn test_run_pbs_no_config() { + let mut cmd = Command::new(cargo::cargo_bin!()); + cmd.args(["pbs"]).assert().failure().stderr(predicates::str::contains("CB_CONFIG is not set")); +} + +/// Tests that Signer runs without CB_CONFIG being set and complains normally +#[test] +fn test_run_signer_no_config() { + let mut cmd = Command::new(cargo::cargo_bin!()); + cmd.args(["signer"]) + .assert() + .failure() + .stderr(predicates::str::contains("CB_CONFIG is not set")); +} From e21fceba45adf0f811d8199096f09e7bf40a5236 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Fri, 6 Mar 2026 13:41:32 -0800 Subject: [PATCH 11/22] one-click kurtosis support --- .gitignore | 3 + justfile | 36 +++++++++++ provisioning/kurtosis-config.yml | 104 +++++++++++++++++++++++++++++++ provisioning/pectra-config.yml | 56 ----------------- 4 files changed, 143 insertions(+), 56 deletions(-) create mode 100644 provisioning/kurtosis-config.yml delete mode 100644 provisioning/pectra-config.yml diff --git a/.gitignore b/.gitignore index d49aa37a..739e111a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ devenv.* devenv.lock .devenv.flake.nix .envrc + +# Generated from testnet +kurtosis-dump \ No newline at end of file diff --git a/justfile b/justfile index b39ab9f4..738cd8fa 100644 --- a/justfile +++ b/justfile @@ -166,3 +166,39 @@ clean: # Runs the suite of tests for all commit-boost crates. test: cargo test --all-features + +# ================= +# === Kurtosis === +# ================= + +# Tear down and clean up all enclaves +clean-kurtosis: + kurtosis clean -a + +# Clean all enclaves and restart the testnet +restart-kurtosis: + just clean-kurtosis + kurtosis run github.com/ethpandaops/ethereum-package \ + --enclave CB-Testnet \ + --args-file provisioning/kurtosis-config.yml + +# Build local docker images and restart testnet +build-kurtosis: + just build-all kurtosis + just restart-kurtosis + +# Inspect running enclave +inspect-kurtosis: + kurtosis enclave inspect CB-Testnet + +# Tail logs for a specific service: just logs-kurtosis +logs-kurtosis service: + kurtosis service logs CB-Testnet {{service}} --follow + +# Shell into a specific service: just shell-kurtosis +shell-kurtosis service: + kurtosis service shell CB-Testnet {{service}} + +# Dump enclave state to disk for post-mortem +dump-kurtosis: + kurtosis enclave dump CB-Testnet ./kurtosis-dump \ No newline at end of file diff --git a/provisioning/kurtosis-config.yml b/provisioning/kurtosis-config.yml new file mode 100644 index 00000000..093534b0 --- /dev/null +++ b/provisioning/kurtosis-config.yml @@ -0,0 +1,104 @@ +# ELs: geth, nethermind, erigon, besu, reth, ethrex +# CLs: nimbus, lighthouse, lodestar, teku, prysm, and grandine +participants: + - el_type: geth + cl_type: nimbus + + - el_type: nethermind + cl_type: lighthouse + + - el_type: erigon + cl_type: lodestar + + - el_type: besu + cl_type: teku + + - el_type: reth + cl_type: prysm + + - el_type: ethrex + cl_type: grandine + +additional_services: + - dora + - spamoor +mev_type: commit-boost + +mev_params: + mev_relay_image: ethpandaops/mev-boost-relay:main + mev_boost_image: commit-boost/pbs:kurtosis + mev_builder_cl_image: sigp/lighthouse:latest + mev_builder_image: ethpandaops/reth-rbuilder:develop + +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + slot_duration_ms: 12000 + num_validator_keys_per_node: 128 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + preregistered_validator_count: 0 + additional_mnemonics: [] + genesis_delay: 20 + genesis_time: 0 + genesis_gaslimit: 60000000 + max_per_epoch_activation_churn_limit: 8 + churn_limit_quotient: 65536 + ejection_balance: 16000000000 + eth1_follow_distance: 2048 + min_validator_withdrawability_delay: 256 + shard_committee_period: 256 + attestation_due_bps_gloas: 2500 + aggregate_due_bps_gloas: 5000 + sync_message_due_bps_gloas: 2500 + contribution_due_bps_gloas: 5000 + payload_attestation_due_bps: 7500 + view_freeze_cutoff_bps: 7500 + inclusion_list_submission_due_bps: 6667 + proposer_inclusion_list_cutoff_bps: 9167 + deneb_fork_epoch: 0 + electra_fork_epoch: 0 + fulu_fork_epoch: 0 + gloas_fork_epoch: 18446744073709551615 + network_sync_base_url: https://snapshots.ethpandaops.io/ + force_snapshot_sync: false + data_column_sidecar_subnet_count: 128 + samples_per_slot: 8 + custody_requirement: 4 + max_blobs_per_block_electra: 9 + max_request_blocks_deneb: 128 + max_request_blob_sidecars_electra: 1152 + target_blobs_per_block_electra: 6 + base_fee_update_fraction_electra: 5007716 + additional_preloaded_contracts: {} + devnet_repo: ethpandaops + prefunded_accounts: {} + bpo_1_epoch: 0 + bpo_1_max_blobs: 15 + bpo_1_target_blobs: 10 + bpo_1_base_fee_update_fraction: 8346193 + bpo_2_epoch: 18446744073709551615 + bpo_2_max_blobs: 21 + bpo_2_target_blobs: 14 + bpo_2_base_fee_update_fraction: 11684671 + bpo_3_epoch: 18446744073709551615 + bpo_3_max_blobs: 0 + bpo_3_target_blobs: 0 + bpo_3_base_fee_update_fraction: 0 + bpo_4_epoch: 18446744073709551615 + bpo_4_max_blobs: 0 + bpo_4_target_blobs: 0 + bpo_4_base_fee_update_fraction: 0 + bpo_5_epoch: 18446744073709551615 + bpo_5_max_blobs: 0 + bpo_5_target_blobs: 0 + bpo_5_base_fee_update_fraction: 0 + withdrawal_type: "0x00" + withdrawal_address: "0x8943545177806ED17B9F23F0a21ee5948eCaa776" + validator_balance: 32 + min_epochs_for_data_column_sidecars_requests: 4096 + min_epochs_for_block_requests: 33024 \ No newline at end of file diff --git a/provisioning/pectra-config.yml b/provisioning/pectra-config.yml deleted file mode 100644 index a78d55a2..00000000 --- a/provisioning/pectra-config.yml +++ /dev/null @@ -1,56 +0,0 @@ -participants: - - el_type: geth - el_image: ethpandaops/geth:prague-devnet-6-c070db6 - el_extra_params: ["--miner.extradata=pawanRocks"] - cl_type: lighthouse - cl_image: ethpandaops/lighthouse:unstable-95cec45 - - el_type: geth - el_image: ethpandaops/geth:prague-devnet-6-c070db6 - el_extra_params: ["--miner.extradata=TekuFromLocal"] - cl_type: teku - cl_image: consensys/teku:develop - - el_type: geth - el_image: ethpandaops/geth:prague-devnet-6-c070db6 - el_extra_params: ["--miner.extradata=lodestarFromLocal"] - cl_type: lodestar - cl_image: ethpandaops/lodestar:unstable-7982031 - - el_type: geth - el_image: ethpandaops/geth:prague-devnet-6-c070db6 - el_extra_params: ["--miner.extradata=prysmFromLocal"] - cl_type: prysm - cl_image: ethpandaops/prysm-beacon-chain:develop-910609a - vc_image: ethpandaops/prysm-validator:develop-910609a - - el_type: geth - el_image: ethpandaops/geth:prague-devnet-6-c070db6 - el_extra_params: ["--miner.extradata=nimbusFromLocal"] - cl_type: nimbus - cl_image: ethpandaops/nimbus-eth2:unstable-dec1cd3 - - el_type: geth - el_image: ethpandaops/geth:prague-devnet-6-c070db6 - el_extra_params: ["--miner.extradata=grandineFromLocal"] - cl_type: grandine - cl_image: ethpandaops/grandine:devnet5-7ba51a4 - -additional_services: - - dora - - tx_spammer - - spamoor_blob -mev_type: commit-boost - -mev_params: - mev_relay_image: jtraglia/mev-boost-relay:electra - mev_boost_image: commitboost_pbs_default # build this locally with scripts/build_local_images.sh - mev_builder_cl_image: ethpandaops/lighthouse:unstable-a1b7d61 - mev_builder_image: ethpandaops/reth-rbuilder:devnet6-fdeb4d6 - -network_params: - electra_fork_epoch: 1 - min_validator_withdrawability_delay: 1 - shard_committee_period: 1 - churn_limit_quotient: 16 - genesis_delay: 120 - -spamoor_blob_params: - throughput: 10 - max_blobs: 2 - max_pending: 40 \ No newline at end of file From 166a83023c0b11e8d515104b631f07ef5bb011f8 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 9 Mar 2026 12:05:58 -0700 Subject: [PATCH 12/22] More doc changes to use new binary --- docs/docs/get_started/running/binary.md | 2 +- docs/docs/get_started/running/metrics.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/get_started/running/binary.md b/docs/docs/get_started/running/binary.md index 10815d6e..74a09373 100644 --- a/docs/docs/get_started/running/binary.md +++ b/docs/docs/get_started/running/binary.md @@ -57,7 +57,7 @@ Modules might also have additional envs required, which should be detailed by th After creating the `cb-config.toml` file, setup the required envs and run the binary. For example: ```bash -CB_CONFIG=./cb-config.toml commit-boost-pbs +CB_CONFIG=./cb-config.toml commit-boost pbs ``` ## Security diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index 38e8534b..58200195 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -12,7 +12,7 @@ Make sure to add the `[metrics]` section to your config file: [metrics] enabled = true ``` -If the section is missing, metrics collection will be disabled. If you generated the `docker-compose.yml` file with `commit-boost-cli`, metrics ports will be automatically configured, and a sample `target.json` file will be created. If you're running the binaries directly, you will need to set the correct environment variables, as described in the [previous section](/get_started/running/binary#common). +If the section is missing, metrics collection will be disabled. If you generated the `docker-compose.yml` file with `commit-boost init`, metrics ports will be automatically configured, and a sample `target.json` file will be created. If you're running the binaries directly, you will need to set the correct environment variables, as described in the [previous section](/get_started/running/binary#common). ## Example setup From 0fa7964d3bc814e40b0ffa5f3eed1b31c798e108 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 9 Mar 2026 15:10:48 -0700 Subject: [PATCH 13/22] Remove panic!() and unwrap() from docker_init.rs and improve test coverage for these flows --- Cargo.lock | 4 + bin/Cargo.toml | 2 + bin/commit-boost.rs | 4 +- bin/tests/binary.rs | 181 ++++++++- crates/cli/Cargo.toml | 4 + crates/cli/src/docker_init.rs | 711 ++++++++++++++++++++++++++++++++-- justfile | 9 +- 7 files changed, 877 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a991b18b..8078e1e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1626,6 +1626,8 @@ dependencies = [ "eyre", "indexmap 2.11.4", "serde_yaml", + "tempfile", + "toml", ] [[package]] @@ -1920,6 +1922,8 @@ dependencies = [ "color-eyre", "eyre", "predicates", + "serde_yaml", + "tempfile", "tokio", "tracing", "tree_hash", diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 0a3395a9..e7a25091 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -21,6 +21,8 @@ tree_hash_derive.workspace = true [dev-dependencies] assert_cmd.workspace = true predicates.workspace = true +serde_yaml.workspace = true +tempfile.workspace = true [[bin]] name = "commit-boost" diff --git a/bin/commit-boost.rs b/bin/commit-boost.rs index 83ff247a..b8dc2261 100644 --- a/bin/commit-boost.rs +++ b/bin/commit-boost.rs @@ -48,11 +48,11 @@ enum Commands { async fn main() -> Result<()> { // Parse the CLI arguments (currently only used for version info, more can be // added later) - let _cli = Cli::parse(); + let cli = Cli::parse(); color_eyre::install()?; - match _cli.command { + match cli.command { Commands::Pbs => run_pbs_service().await?, Commands::Signer => run_signer_service().await?, Commands::Init { config_path, output_path } => run_init(config_path, output_path).await?, diff --git a/bin/tests/binary.rs b/bin/tests/binary.rs index 8c524c8f..c000ed11 100644 --- a/bin/tests/binary.rs +++ b/bin/tests/binary.rs @@ -1,19 +1,76 @@ use assert_cmd::{Command, cargo}; +use cb_cli::docker_init::{CB_COMPOSE_FILE, CB_ENV_FILE}; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const MINIMAL_PBS_TOML: &str = r#" +chain = "Holesky" +[pbs] +docker_image = "ghcr.io/commit-boost/pbs:latest" +"#; + +const MINIMAL_WITH_MODULE_TOML: &str = r#" +chain = "Holesky" +[pbs] +docker_image = "ghcr.io/commit-boost/pbs:latest" + +[signer.local.loader] +key_path = "/keys/keys.json" + +[[modules]] +id = "DA_COMMIT" +type = "commit" +docker_image = "test_da_commit" +"#; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Returns a `Command` pointed at the `commit-boost` binary under test. +fn cmd() -> Command { + Command::new(cargo::cargo_bin!()) +} + +/// Writes `contents` to `cb.toml` inside `dir` and returns the path. +fn write_config(dir: &tempfile::TempDir, contents: &str) -> std::path::PathBuf { + let path = dir.path().join("cb.toml"); + std::fs::write(&path, contents).expect("write test config"); + path +} + +/// Returns a `commit-boost init` command configured with the given config and +/// output directory. +fn init_cmd(config: &std::path::Path, output_dir: &std::path::Path) -> Command { + let mut c = cmd(); + c.args([ + "init", + "--config", + config.to_str().expect("valid config path"), + "--output", + output_dir.to_str().expect("valid output dir"), + ]); + c +} + +// --------------------------------------------------------------------------- +// Binary smoke tests +// --------------------------------------------------------------------------- /// Tests that the binary can be run and returns a version string #[test] fn test_load_example_config() { - let mut cmd = Command::new(cargo::cargo_bin!()); let expected_version = format!("Commit-Boost {}\n", commit_boost::VERSION); - cmd.arg("--version").assert().success().stdout(expected_version); + cmd().arg("--version").assert().success().stdout(expected_version); } /// Tests that the init command can be run and complains about not having /// --config set #[test] fn test_run_init() { - let mut cmd = Command::new(cargo::cargo_bin!()); - cmd.args(["init"]).assert().failure().stderr(predicates::str::contains( + cmd().args(["init"]).assert().failure().stderr(predicates::str::contains( "error: the following required arguments were not provided:\n --config ", )); } @@ -21,16 +78,124 @@ fn test_run_init() { /// Tests that PBS runs without CB_CONFIG being set and complains normally #[test] fn test_run_pbs_no_config() { - let mut cmd = Command::new(cargo::cargo_bin!()); - cmd.args(["pbs"]).assert().failure().stderr(predicates::str::contains("CB_CONFIG is not set")); + cmd() + .args(["pbs"]) + .assert() + .failure() + .stderr(predicates::str::contains("CB_CONFIG is not set")); } /// Tests that Signer runs without CB_CONFIG being set and complains normally #[test] fn test_run_signer_no_config() { - let mut cmd = Command::new(cargo::cargo_bin!()); - cmd.args(["signer"]) + cmd() + .args(["signer"]) .assert() .failure() .stderr(predicates::str::contains("CB_CONFIG is not set")); } + +// --------------------------------------------------------------------------- +// handle_docker_init (via `commit-boost init`) integration tests +// --------------------------------------------------------------------------- + +/// Minimal PBS-only config produces a compose file and no .env file. +#[test] +fn test_init_pbs_only_creates_compose_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let config = write_config(&dir, MINIMAL_PBS_TOML); + + init_cmd(&config, dir.path()).assert().success(); + + assert!(dir.path().join(CB_COMPOSE_FILE).exists(), "compose file should be created"); + assert!(!dir.path().join(CB_ENV_FILE).exists(), "no .env file for PBS-only config"); +} + +/// PBS-only compose file has the expected service structure. +#[test] +fn test_init_compose_file_pbs_service_structure() { + let dir = tempfile::tempdir().expect("tempdir"); + let config = write_config(&dir, MINIMAL_PBS_TOML); + + init_cmd(&config, dir.path()).assert().success(); + + let contents = + std::fs::read_to_string(dir.path().join(CB_COMPOSE_FILE)).expect("read compose file"); + let compose: serde_yaml::Value = + serde_yaml::from_str(&contents).expect("compose file is valid YAML"); + + let pbs = &compose["services"]["cb_pbs"]; + assert!(!pbs.is_null(), "cb_pbs service must exist"); + assert_eq!(pbs["image"].as_str(), Some("ghcr.io/commit-boost/pbs:latest"), "image"); + assert_eq!(pbs["container_name"].as_str(), Some("cb_pbs"), "container_name"); + + // Config file must be mounted inside the container. + let volumes = pbs["volumes"].as_sequence().expect("volumes is a list"); + assert!( + volumes.iter().any(|v| v.as_str().map_or(false, |s| s.ends_with(":/cb-config.toml:ro"))), + "config must be mounted at /cb-config.toml" + ); + + // Required environment variables must be present. + let env = &pbs["environment"]; + assert!(!env["CB_CONFIG"].is_null(), "CB_CONFIG env var must be set"); + assert!(!env["CB_PBS_ENDPOINT"].is_null(), "CB_PBS_ENDPOINT env var must be set"); + + // Port 18550 must be exposed. + let ports = pbs["ports"].as_sequence().expect("ports is a list"); + assert!( + ports.iter().any(|p| p.as_str().map_or(false, |s| s.contains("18550"))), + "port 18550 must be exposed" + ); + + // No signer service and no extra network in a PBS-only config. + assert!(compose["services"]["cb_signer"].is_null(), "cb_signer must not exist"); + assert!(compose["networks"].is_null(), "no networks for PBS-only config"); +} + +/// Config with a commit module produces both a compose file and a .env file. +#[test] +fn test_init_with_module_creates_env_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let config = write_config(&dir, MINIMAL_WITH_MODULE_TOML); + + init_cmd(&config, dir.path()).assert().success(); + + assert!(dir.path().join(CB_COMPOSE_FILE).exists(), "compose file should be created"); + assert!(dir.path().join(CB_ENV_FILE).exists(), ".env file should be created for modules"); +} + +/// .env file contains a JWT entry for the module. +#[test] +fn test_init_env_file_contains_module_jwt() { + let dir = tempfile::tempdir().expect("tempdir"); + let config = write_config(&dir, MINIMAL_WITH_MODULE_TOML); + + init_cmd(&config, dir.path()).assert().success(); + + let env_contents = + std::fs::read_to_string(dir.path().join(CB_ENV_FILE)).expect("read .env file"); + assert!(env_contents.contains("CB_JWT_DA_COMMIT="), ".env must contain module JWT"); +} + +/// Missing --config argument produces a clear error message. +#[test] +fn test_init_missing_config_flag_fails_with_message() { + cmd().args(["init"]).assert().failure().stderr(predicates::str::contains("--config")); +} + +/// Non-existent config file produces an error. +#[test] +fn test_init_nonexistent_config_file_fails() { + let dir = tempfile::tempdir().expect("tempdir"); + cmd() + .args([ + "init", + "--config", + "/nonexistent/path/cb.toml", + "--output", + dir.path().to_str().expect("valid dir"), + ]) + .assert() + .failure(); +} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 2acc6a7b..3e713397 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -12,3 +12,7 @@ docker-compose-types.workspace = true eyre.workspace = true indexmap.workspace = true serde_yaml.workspace = true + +[dev-dependencies] +tempfile.workspace = true +toml.workspace = true diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 450aa7bd..3cbde28a 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -31,9 +31,9 @@ use eyre::Result; use indexmap::IndexMap; /// Name of the docker compose file -pub(super) const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml"; +pub const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml"; /// Name of the envs file -pub(super) const CB_ENV_FILE: &str = ".cb.env"; +pub const CB_ENV_FILE: &str = ".cb.env"; const SIGNER_NETWORK: &str = "signer_network"; @@ -108,8 +108,13 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re // Get the custom chain spec, if any let chain_spec_path = CommitBoostConfig::chain_spec_file(&config_path); if let Some(spec) = chain_spec_path { - // this is ok since the config has already been loaded once - let filename = spec.file_name().unwrap().to_str().unwrap(); + let filename = spec + .file_name() + .ok_or_else(|| eyre::eyre!("Chain spec path has no filename: {}", spec.display()))? + .to_str() + .ok_or_else(|| { + eyre::eyre!("Chain spec filename is not valid UTF-8: {}", spec.display()) + })?; let chain_spec = ServiceChainSpecInfo { env: get_env_val(CHAIN_SPEC_ENV, &format!("/{filename}")), volume: Volumes::Simple(format!("{}:/{}:ro", spec.display(), filename)), @@ -129,14 +134,15 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re service_config.config_info.cb_config.modules.as_ref().is_some_and(|modules| { modules.iter().any(|module| matches!(module.kind, ModuleKind::Commit)) }); - let signer_config = - if needs_signer_module { - Some(service_config.config_info.cb_config.signer.clone().expect( - "Signer module required but no signer config provided in Commit-Boost config", - )) - } else { - None - }; + let signer_config = if needs_signer_module { + Some(service_config.config_info.cb_config.signer.clone().ok_or_else(|| { + eyre::eyre!( + "Signer module required but no signer config provided in Commit-Boost config" + ) + })?) + } else { + None + }; let signer_server = if let Some(SignerConfig { inner: SignerType::Remote { url }, .. }) = &service_config.config_info.cb_config.signer { @@ -190,7 +196,9 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re services.insert("cb_signer".to_owned(), Some(signer_service)); } SignerType::Remote { .. } => { - panic!("Signer module required but remote config provided"); + eyre::bail!( + "Signer module required but remote signer config provided; use a local or Dirk signer instead" + ); } } } @@ -295,7 +303,7 @@ fn create_pbs_service(service_config: &mut ServiceCreationInfo) -> eyre::Result< } // Create the service - volumes.extend(get_log_volume(&cb_config.logs, PBS_SERVICE_NAME)); + volumes.extend(get_log_volume(&cb_config.logs, PBS_SERVICE_NAME)?); let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), image: Some(cb_config.pbs.docker_image.clone()), @@ -370,7 +378,7 @@ fn create_signer_service_local( let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); envs.insert(key, val); } - volumes.extend(get_log_volume(&cb_config.logs, SIGNER_SERVICE_NAME)); + volumes.extend(get_log_volume(&cb_config.logs, SIGNER_SERVICE_NAME)?); // write jwts to env service_config.envs.insert(JWTS_ENV.into(), format_comma_separated(&service_config.jwts)); @@ -523,7 +531,7 @@ fn create_signer_service_dirk( let (key, val) = get_env_val(LOGS_DIR_ENV, LOGS_DIR_DEFAULT); envs.insert(key, val); } - volumes.extend(get_log_volume(&cb_config.logs, SIGNER_SERVICE_NAME)); + volumes.extend(get_log_volume(&cb_config.logs, SIGNER_SERVICE_NAME)?); // write jwts to env service_config.envs.insert(JWTS_ENV.into(), format_comma_separated(&service_config.jwts)); @@ -547,7 +555,7 @@ fn create_signer_service_dirk( envs.insert(key, val); } Some(ProxyStore::ERC2335 { .. }) => { - panic!("ERC2335 store not supported with Dirk signer"); + eyre::bail!("ERC2335 proxy store is not supported with the Dirk signer"); } None => {} } @@ -615,7 +623,7 @@ fn create_module_service( // volumes let mut module_volumes = vec![config_volume.clone()]; - module_volumes.extend(get_log_volume(&cb_config.logs, &module.id)); + module_volumes.extend(get_log_volume(&cb_config.logs, &module.id)?); // Chain spec env/volume if let Some(spec) = &service_config.chain_spec { @@ -745,18 +753,667 @@ fn get_env_uval(k: &str, v: u64) -> (String, Option) { // (k.into(), Some(SingleValue::Bool(v))) // } -fn get_log_volume(config: &LogsSettings, module_id: &str) -> Option { - config.file.enabled.then_some({ - let p = config.file.dir_path.join(module_id.to_lowercase()); - Volumes::Simple(format!( - "{}:{}", - p.to_str().expect("could not convert pathbuf to str"), - LOGS_DIR_DEFAULT - )) - }) +fn get_log_volume(config: &LogsSettings, module_id: &str) -> eyre::Result> { + if !config.file.enabled { + return Ok(None); + } + let p = config.file.dir_path.join(module_id.to_lowercase()); + let host_path = p + .to_str() + .ok_or_else(|| eyre::eyre!("Log directory path is not valid UTF-8: {}", p.display()))?; + Ok(Some(Volumes::Simple(format!("{host_path}:{LOGS_DIR_DEFAULT}")))) } /// Formats as a comma separated list of key=value fn format_comma_separated(map: &IndexMap) -> String { map.iter().map(|(k, v)| format!("{k}={v}")).collect::>().join(",") } + +#[cfg(test)] +mod tests { + use cb_common::{ + config::{ + CommitBoostConfig, FileLogSettings, LogsSettings, MetricsConfig, StdoutLogSettings, + }, + signer::{ProxyStore, SignerLoader}, + }; + use docker_compose_types::{Environment, Ports, SingleValue, Volumes}; + + use super::*; + + // ------------------------------------------------------------------------- + // Shared test fixtures + // ------------------------------------------------------------------------- + + fn logs_disabled() -> LogsSettings { + LogsSettings::default() + } + + fn logs_enabled(dir: &str) -> LogsSettings { + LogsSettings { + stdout: StdoutLogSettings::default(), + file: FileLogSettings { + enabled: true, + dir_path: dir.into(), + ..FileLogSettings::default() + }, + } + } + + /// Deserialize a minimal PBS-only `CommitBoostConfig` from inline TOML. + /// No relays, so `validate()` won't make network calls. + fn minimal_cb_config() -> CommitBoostConfig { + toml::from_str( + r#" + chain = "Holesky" + [pbs] + docker_image = "ghcr.io/commit-boost/pbs:latest" + "#, + ) + .expect("valid minimal test config") + } + + fn minimal_service_config() -> ServiceCreationInfo { + ServiceCreationInfo { + config_info: CommitBoostConfigInfo { + cb_config: minimal_cb_config(), + config_volume: Volumes::Simple("./cb.toml:/cb.toml:ro".into()), + }, + envs: IndexMap::new(), + targets: Vec::new(), + warnings: Vec::new(), + jwts: IndexMap::new(), + chain_spec: None, + metrics_port: 9100, + } + } + + fn metrics_config() -> MetricsConfig { + MetricsConfig { + enabled: true, + host: "127.0.0.1".parse().expect("valid IP"), + start_port: 9100, + } + } + + // ------------------------------------------------------------------------- + // Service inspection helpers + // ------------------------------------------------------------------------- + + fn env_str(service: &Service, key: &str) -> Option { + match &service.environment { + Environment::KvPair(map) => map.get(key).and_then(|v| match v { + Some(SingleValue::String(s)) => Some(s.clone()), + _ => None, + }), + _ => None, + } + } + + fn env_u64(service: &Service, key: &str) -> Option { + match &service.environment { + Environment::KvPair(map) => map.get(key).and_then(|v| match v { + Some(SingleValue::Unsigned(n)) => Some(*n), + _ => None, + }), + _ => None, + } + } + + fn has_env_key(service: &Service, key: &str) -> bool { + match &service.environment { + Environment::KvPair(map) => map.contains_key(key), + _ => false, + } + } + + fn has_volume(service: &Service, substr: &str) -> bool { + service.volumes.iter().any(|v| matches!(v, Volumes::Simple(s) if s.contains(substr))) + } + + fn has_port(service: &Service, substr: &str) -> bool { + match &service.ports { + Ports::Short(ports) => ports.iter().any(|p| p.contains(substr)), + _ => false, + } + } + + // --- get_env_val --- + + #[test] + fn test_get_env_val_returns_string_pair() { + let (key, val) = get_env_val("MY_KEY", "my_value"); + assert_eq!(key, "MY_KEY"); + assert_eq!(val, Some(SingleValue::String("my_value".into()))); + } + + #[test] + fn test_get_env_val_empty_value() { + let (key, val) = get_env_val("EMPTY", ""); + assert_eq!(key, "EMPTY"); + assert_eq!(val, Some(SingleValue::String("".into()))); + } + + // --- get_env_uval --- + + #[test] + fn test_get_env_uval_returns_unsigned_pair() { + let (key, val) = get_env_uval("PORT", 9100); + assert_eq!(key, "PORT"); + assert_eq!(val, Some(SingleValue::Unsigned(9100))); + } + + // --- get_env_same --- + + #[test] + fn test_get_env_same_interpolates_self() { + let (key, val) = get_env_same("JWTS_ENV"); + assert_eq!(key, "JWTS_ENV"); + assert_eq!(val, Some(SingleValue::String("${JWTS_ENV}".into()))); + } + + // --- get_env_interp --- + + #[test] + fn test_get_env_interp_different_key_and_var() { + let (key, val) = get_env_interp("MODULE_JWT_ENV", "CB_JWT_MY_MODULE"); + assert_eq!(key, "MODULE_JWT_ENV"); + assert_eq!(val, Some(SingleValue::String("${CB_JWT_MY_MODULE}".into()))); + } + + // --- format_comma_separated --- + + #[test] + fn test_format_comma_separated_empty() { + let map = IndexMap::new(); + assert_eq!(format_comma_separated(&map), ""); + } + + #[test] + fn test_format_comma_separated_single_entry() { + let mut map = IndexMap::new(); + map.insert(ModuleId::from("module_a".to_owned()), "secret123".into()); + assert_eq!(format_comma_separated(&map), "module_a=secret123"); + } + + #[test] + fn test_format_comma_separated_multiple_entries_preserves_order() { + let mut map = IndexMap::new(); + map.insert(ModuleId::from("module_a".to_owned()), "jwt_a".into()); + map.insert(ModuleId::from("module_b".to_owned()), "jwt_b".into()); + map.insert(ModuleId::from("module_c".to_owned()), "jwt_c".into()); + assert_eq!(format_comma_separated(&map), "module_a=jwt_a,module_b=jwt_b,module_c=jwt_c"); + } + + // --- get_log_volume --- + + #[test] + fn test_get_log_volume_disabled_returns_none() -> eyre::Result<()> { + let logs = logs_disabled(); + let result = get_log_volume(&logs, "cb_pbs")?; + assert!(result.is_none()); + Ok(()) + } + + #[test] + fn test_get_log_volume_enabled_returns_correct_volume() -> eyre::Result<()> { + let logs = logs_enabled("/var/log/commit-boost"); + let result = get_log_volume(&logs, "cb_pbs")?; + let volume = result.expect("expected a volume when file logging is enabled"); + assert_eq!( + volume, + Volumes::Simple(format!("/var/log/commit-boost/cb_pbs:{LOGS_DIR_DEFAULT}")) + ); + Ok(()) + } + + #[test] + fn test_get_log_volume_lowercases_module_id() -> eyre::Result<()> { + let logs = logs_enabled("/logs"); + let result = get_log_volume(&logs, "MY_MODULE")?; + let volume = result.expect("expected a volume when file logging is enabled"); + assert_eq!(volume, Volumes::Simple(format!("/logs/my_module:{LOGS_DIR_DEFAULT}"))); + Ok(()) + } + + #[test] + fn test_get_log_volume_enabled_with_nested_dir() -> eyre::Result<()> { + let logs = logs_enabled("/home/user/cb/logs"); + let result = get_log_volume(&logs, "cb_signer")?; + let volume = result.expect("expected a volume when file logging is enabled"); + assert_eq!( + volume, + Volumes::Simple(format!("/home/user/cb/logs/cb_signer:{LOGS_DIR_DEFAULT}")) + ); + Ok(()) + } + + // ------------------------------------------------------------------------- + // write_env_file + // ------------------------------------------------------------------------- + + #[test] + fn test_write_env_file_empty_map() -> eyre::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join(".cb.env"); + write_env_file(&IndexMap::new(), &path)?; + let contents = std::fs::read_to_string(&path)?; + assert_eq!(contents, ""); + Ok(()) + } + + #[test] + fn test_write_env_file_single_entry() -> eyre::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join(".cb.env"); + let mut map = IndexMap::new(); + map.insert("MY_KEY".to_owned(), "my_value".to_owned()); + write_env_file(&map, &path)?; + let contents = std::fs::read_to_string(&path)?; + assert_eq!(contents, "MY_KEY=my_value\n"); + Ok(()) + } + + #[test] + fn test_write_env_file_multiple_entries_preserves_order() -> eyre::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join(".cb.env"); + let mut map = IndexMap::new(); + map.insert("KEY_A".to_owned(), "val_a".to_owned()); + map.insert("KEY_B".to_owned(), "val_b".to_owned()); + map.insert("KEY_C".to_owned(), "val_c".to_owned()); + write_env_file(&map, &path)?; + let contents = std::fs::read_to_string(&path)?; + assert_eq!(contents, "KEY_A=val_a\nKEY_B=val_b\nKEY_C=val_c\n"); + Ok(()) + } + + // ------------------------------------------------------------------------- + // write_compose_file + // ------------------------------------------------------------------------- + + #[test] + fn test_write_compose_file_creates_valid_yaml() -> eyre::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join(CB_COMPOSE_FILE); + let compose = docker_compose_types::Compose::default(); + let service_config = minimal_service_config(); + write_compose_file(&compose, &path, &service_config)?; + assert!(path.exists()); + let contents = std::fs::read_to_string(&path)?; + assert!(!contents.is_empty()); + Ok(()) + } + + // ------------------------------------------------------------------------- + // create_pbs_service + // ------------------------------------------------------------------------- + + #[test] + fn test_create_pbs_service_basic() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let service = create_pbs_service(&mut sc)?; + + assert_eq!(service.container_name.as_deref(), Some("cb_pbs")); + assert_eq!(service.image.as_deref(), Some("ghcr.io/commit-boost/pbs:latest")); + assert!(env_str(&service, CONFIG_ENV).is_some()); + assert!(env_str(&service, PBS_ENDPOINT_ENV).is_some()); + assert!(service.healthcheck.is_some()); + Ok(()) + } + + #[test] + fn test_create_pbs_service_exposes_pbs_port() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let service = create_pbs_service(&mut sc)?; + // Default PBS port is 18550 + assert!(has_port(&service, "18550")); + Ok(()) + } + + #[test] + fn test_create_pbs_service_with_metrics() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + sc.config_info.cb_config.metrics = Some(metrics_config()); + sc.metrics_port = 9100; + let service = create_pbs_service(&mut sc)?; + + assert_eq!(env_u64(&service, METRICS_PORT_ENV), Some(9100)); + assert!(has_port(&service, "9100")); + // port counter incremented + assert_eq!(sc.metrics_port, 9101); + // target added for prometheus + assert!(!sc.targets.is_empty()); + Ok(()) + } + + #[test] + fn test_create_pbs_service_with_file_logging() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + sc.config_info.cb_config.logs = logs_enabled("/var/log/cb"); + let service = create_pbs_service(&mut sc)?; + + assert!(env_str(&service, LOGS_DIR_ENV).is_some()); + assert!(has_volume(&service, "pbs")); + Ok(()) + } + + #[test] + fn test_create_pbs_service_with_chain_spec() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + sc.chain_spec = Some(ServiceChainSpecInfo { + env: get_env_val(CHAIN_SPEC_ENV, "/chain.json"), + volume: Volumes::Simple("/host/chain.json:/chain.json:ro".into()), + }); + let service = create_pbs_service(&mut sc)?; + + assert_eq!(env_str(&service, CHAIN_SPEC_ENV), Some("/chain.json".into())); + assert!(has_volume(&service, "chain.json")); + Ok(()) + } + + #[test] + fn test_create_pbs_service_no_metrics_no_metrics_env() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let service = create_pbs_service(&mut sc)?; + assert!(!has_env_key(&service, METRICS_PORT_ENV)); + Ok(()) + } + + // ------------------------------------------------------------------------- + // create_signer_service_local + // ------------------------------------------------------------------------- + + fn local_signer_config() -> SignerConfig { + toml::from_str( + r#" + [local.loader] + key_path = "/keys/keys.json" + "#, + ) + .expect("valid local signer config") + } + + #[test] + fn test_create_signer_service_local_file_loader() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = local_signer_config(); + let loader = SignerLoader::File { key_path: "/keys/keys.json".into() }; + let service = create_signer_service_local(&mut sc, &signer_config, &loader, &None)?; + + assert_eq!(service.container_name.as_deref(), Some("cb_signer")); + assert!(env_str(&service, SIGNER_KEYS_ENV).is_some()); + assert!(has_volume(&service, "keys.json")); + Ok(()) + } + + #[test] + fn test_create_signer_service_local_validators_dir_loader() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = local_signer_config(); + let loader = SignerLoader::ValidatorsDir { + keys_path: "/keys".into(), + secrets_path: "/secrets".into(), + format: cb_common::signer::ValidatorKeysFormat::Lighthouse, + }; + let service = create_signer_service_local(&mut sc, &signer_config, &loader, &None)?; + + assert!(env_str(&service, SIGNER_DIR_KEYS_ENV).is_some()); + assert!(env_str(&service, SIGNER_DIR_SECRETS_ENV).is_some()); + assert!(has_volume(&service, "/keys")); + assert!(has_volume(&service, "/secrets")); + Ok(()) + } + + #[test] + fn test_create_signer_service_local_with_file_proxy_store() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = local_signer_config(); + let loader = SignerLoader::File { key_path: "/keys/keys.json".into() }; + let store = Some(ProxyStore::File { proxy_dir: "/proxies".into() }); + let service = create_signer_service_local(&mut sc, &signer_config, &loader, &store)?; + + assert!(env_str(&service, PROXY_DIR_ENV).is_some()); + assert!(has_volume(&service, "/proxies")); + Ok(()) + } + + #[test] + fn test_create_signer_service_local_with_erc2335_proxy_store() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = local_signer_config(); + let loader = SignerLoader::File { key_path: "/keys/keys.json".into() }; + let store = Some(ProxyStore::ERC2335 { + keys_path: "/proxy/keys".into(), + secrets_path: "/proxy/secrets".into(), + }); + let service = create_signer_service_local(&mut sc, &signer_config, &loader, &store)?; + + assert!(env_str(&service, PROXY_DIR_KEYS_ENV).is_some()); + assert!(env_str(&service, PROXY_DIR_SECRETS_ENV).is_some()); + assert!(has_volume(&service, "/proxy/keys")); + assert!(has_volume(&service, "/proxy/secrets")); + Ok(()) + } + + #[test] + fn test_create_signer_service_local_jwts_written_to_envs() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + sc.jwts.insert(ModuleId::from("MY_MODULE".to_owned()), "jwt_secret_abc".into()); + let signer_config = local_signer_config(); + let loader = SignerLoader::File { key_path: "/keys/keys.json".into() }; + create_signer_service_local(&mut sc, &signer_config, &loader, &None)?; + + // JWTS_ENV written as comma-separated to service_config.envs + let jwts_val = sc.envs.get(JWTS_ENV).expect("JWTS_ENV must be set in envs"); + assert!(jwts_val.contains("MY_MODULE=jwt_secret_abc")); + Ok(()) + } + + // ------------------------------------------------------------------------- + // create_signer_service_dirk + // ------------------------------------------------------------------------- + + fn dirk_signer_config() -> SignerConfig { + toml::from_str( + r#" + docker_image = "commitboost_signer" + [dirk] + cert_path = "/certs/client.crt" + key_path = "/certs/client.key" + secrets_path = "/dirk_secrets" + [[dirk.hosts]] + url = "https://gateway.dirk.url" + wallets = ["wallet1"] + "#, + ) + .expect("valid dirk signer config") + } + + #[test] + fn test_create_signer_service_dirk_basic() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = dirk_signer_config(); + let service = create_signer_service_dirk( + &mut sc, + &signer_config, + Path::new("/certs/client.crt"), + Path::new("/certs/client.key"), + Path::new("/dirk_secrets"), + &None, + &None, + )?; + + assert_eq!(service.container_name.as_deref(), Some("cb_signer")); + assert!(env_str(&service, DIRK_CERT_ENV).is_some()); + assert!(env_str(&service, DIRK_KEY_ENV).is_some()); + assert!(env_str(&service, DIRK_DIR_SECRETS_ENV).is_some()); + assert!(has_volume(&service, "client.crt")); + assert!(has_volume(&service, "client.key")); + assert!(has_volume(&service, "dirk_secrets")); + Ok(()) + } + + #[test] + fn test_create_signer_service_dirk_with_ca_cert() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = dirk_signer_config(); + let ca_cert = Some(PathBuf::from("/certs/ca.crt")); + let service = create_signer_service_dirk( + &mut sc, + &signer_config, + Path::new("/certs/client.crt"), + Path::new("/certs/client.key"), + Path::new("/dirk_secrets"), + &ca_cert, + &None, + )?; + + assert!(env_str(&service, DIRK_CA_CERT_ENV).is_some()); + assert!(has_volume(&service, "ca.crt")); + Ok(()) + } + + #[test] + fn test_create_signer_service_dirk_without_ca_cert() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = dirk_signer_config(); + let service = create_signer_service_dirk( + &mut sc, + &signer_config, + Path::new("/certs/client.crt"), + Path::new("/certs/client.key"), + Path::new("/dirk_secrets"), + &None, + &None, + )?; + + assert!(!has_env_key(&service, DIRK_CA_CERT_ENV)); + assert!(!has_volume(&service, "ca.crt")); + Ok(()) + } + + #[test] + fn test_create_signer_service_dirk_with_file_proxy_store() -> eyre::Result<()> { + let mut sc = minimal_service_config(); + let signer_config = dirk_signer_config(); + let store = Some(ProxyStore::File { proxy_dir: "/proxies".into() }); + let service = create_signer_service_dirk( + &mut sc, + &signer_config, + Path::new("/certs/client.crt"), + Path::new("/certs/client.key"), + Path::new("/dirk_secrets"), + &None, + &store, + )?; + + assert!(env_str(&service, PROXY_DIR_ENV).is_some()); + assert!(has_volume(&service, "/proxies")); + Ok(()) + } + + #[test] + fn test_create_signer_service_dirk_erc2335_store_returns_error() { + let mut sc = minimal_service_config(); + let signer_config = dirk_signer_config(); + let store = Some(ProxyStore::ERC2335 { + keys_path: "/proxy/keys".into(), + secrets_path: "/proxy/secrets".into(), + }); + let result = create_signer_service_dirk( + &mut sc, + &signer_config, + Path::new("/certs/client.crt"), + Path::new("/certs/client.key"), + Path::new("/dirk_secrets"), + &None, + &store, + ); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("ERC2335")); + } + + // ------------------------------------------------------------------------- + // create_module_service + // ------------------------------------------------------------------------- + + fn commit_module() -> StaticModuleConfig { + toml::from_str( + r#" + id = "DA_COMMIT" + type = "commit" + docker_image = "test_da_commit" + "#, + ) + .expect("valid module config") + } + + #[test] + fn test_create_module_service_container_name_format() -> eyre::Result<()> { + let module = commit_module(); + let mut sc = minimal_service_config(); + let (cid, _) = create_module_service(&module, "http://cb_signer:20000", &mut sc)?; + assert_eq!(cid, "cb_da_commit"); + Ok(()) + } + + #[test] + fn test_create_module_service_sets_required_envs() -> eyre::Result<()> { + let module = commit_module(); + let mut sc = minimal_service_config(); + let (_, service) = create_module_service(&module, "http://cb_signer:20000", &mut sc)?; + + assert!(env_str(&service, MODULE_ID_ENV).is_some()); + assert!(env_str(&service, CONFIG_ENV).is_some()); + assert!(env_str(&service, SIGNER_URL_ENV) == Some("http://cb_signer:20000".into())); + Ok(()) + } + + #[test] + fn test_create_module_service_jwt_written_to_service_config_envs() -> eyre::Result<()> { + let module = commit_module(); + let mut sc = minimal_service_config(); + create_module_service(&module, "http://cb_signer:20000", &mut sc)?; + + // JWT env var should be in the outer service_config.envs (for .env file) + let jwt_key = format!("CB_JWT_{}", "DA_COMMIT".to_uppercase()); + assert!(sc.envs.contains_key(&jwt_key)); + // and also recorded in jwts map + assert!(sc.jwts.contains_key(&ModuleId::from("DA_COMMIT".to_owned()))); + Ok(()) + } + + #[test] + fn test_create_module_service_custom_env_forwarded() -> eyre::Result<()> { + let mut module = commit_module(); + let mut env_map = std::collections::HashMap::new(); + env_map.insert("SOME_ENV_VAR".to_owned(), "some_value".to_owned()); + module.env = Some(env_map); + + let mut sc = minimal_service_config(); + let (_, service) = create_module_service(&module, "http://cb_signer:20000", &mut sc)?; + + assert_eq!(env_str(&service, "SOME_ENV_VAR"), Some("some_value".into())); + Ok(()) + } + + #[test] + fn test_create_module_service_depends_on_signer() -> eyre::Result<()> { + let module = commit_module(); + let mut sc = minimal_service_config(); + let (_, service) = create_module_service(&module, "http://cb_signer:20000", &mut sc)?; + + match &service.depends_on { + docker_compose_types::DependsOnOptions::Conditional(deps) => { + assert!(deps.contains_key("cb_signer")); + } + docker_compose_types::DependsOnOptions::Simple(deps) => { + // Remote signer path returns empty depends_on — but this is a local signer + // config (signer is None), so it still depends on cb_signer + assert!(deps.is_empty(), "unexpected empty depends_on for local signer"); + } + } + Ok(()) + } +} diff --git a/justfile b/justfile index 738cd8fa..1e16321c 100644 --- a/justfile +++ b/justfile @@ -11,6 +11,13 @@ fmt-check: clippy: cargo +{{toolchain}} clippy --all-features --no-deps -- -D warnings +# Everything needed to run before pushing +checklist: + cargo check + just fmt + just clippy + just test + # =================================== # === Build Commands for Services === # =================================== @@ -201,4 +208,4 @@ shell-kurtosis service: # Dump enclave state to disk for post-mortem dump-kurtosis: - kurtosis enclave dump CB-Testnet ./kurtosis-dump \ No newline at end of file + kurtosis enclave dump CB-Testnet ./kurtosis-dump From 957d394b9c3ecb69950c6abf2368524b97163780 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 10 Mar 2026 10:34:11 -0700 Subject: [PATCH 14/22] Add cmds for test coverage + regroup kurtosis commands for easier viewing in `just -l` --- justfile | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/justfile b/justfile index 1e16321c..eb6dc69f 100644 --- a/justfile +++ b/justfile @@ -174,38 +174,59 @@ clean: test: cargo test --all-features +# ===================== +# === Test Coverage === +# ===================== + +# Generate an HTML test coverage report and open it in the browser. +# Recompiles the workspace with LLVM coverage instrumentation, runs all tests, +# and writes the report to target/llvm-cov/html/index.html. +# Incremental recompilation works normally — no need to clean between runs. +# If results look wrong after upgrading cargo-llvm-cov, run `just coverage-clean` first. +# Requires: cargo install cargo-llvm-cov && rustup component add llvm-tools-preview +coverage: + cargo llvm-cov --all-features --html --open + +# Print a quick coverage summary to the terminal without opening a browser. +coverage-summary: + cargo llvm-cov --all-features --summary-only + +# Remove all coverage instrumentation artifacts produced by cargo-llvm-cov. +coverage-clean: + cargo llvm-cov clean --workspace + # ================= # === Kurtosis === # ================= # Tear down and clean up all enclaves -clean-kurtosis: +kurtosis-clean: kurtosis clean -a # Clean all enclaves and restart the testnet -restart-kurtosis: - just clean-kurtosis +kurtosis-restart: + just kurtosis-clean kurtosis run github.com/ethpandaops/ethereum-package \ --enclave CB-Testnet \ --args-file provisioning/kurtosis-config.yml # Build local docker images and restart testnet -build-kurtosis: +kurtosis-build: just build-all kurtosis - just restart-kurtosis + just kurtosis-restart # Inspect running enclave -inspect-kurtosis: +kurtosis-inspect: kurtosis enclave inspect CB-Testnet -# Tail logs for a specific service: just logs-kurtosis -logs-kurtosis service: +# Tail logs for a specific service: just kurtosis-logs +kurtosis-logs service: kurtosis service logs CB-Testnet {{service}} --follow -# Shell into a specific service: just shell-kurtosis -shell-kurtosis service: +# Shell into a specific service: just kurtosis-shell +kurtosis-shell service: kurtosis service shell CB-Testnet {{service}} # Dump enclave state to disk for post-mortem -dump-kurtosis: +kurtosis-dump: kurtosis enclave dump CB-Testnet ./kurtosis-dump From 87ab03e81c83989de02834aa9db9fffa6bd48b49 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Wed, 11 Mar 2026 14:29:35 -0700 Subject: [PATCH 15/22] Add criterion microbenchmark test for get_header with Justfile cmds --- Cargo.lock | 154 +++++++++++++++++++++++++++ Cargo.toml | 1 + benches/microbench/Cargo.toml | 19 ++++ benches/microbench/src/get_header.rs | 148 +++++++++++++++++++++++++ justfile | 39 +++++++ 5 files changed, 361 insertions(+) create mode 100644 benches/microbench/Cargo.toml create mode 100644 benches/microbench/src/get_header.rs diff --git a/Cargo.lock b/Cargo.lock index 8078e1e9..a1ba2329 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -877,6 +877,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.20" @@ -1599,6 +1605,25 @@ dependencies = [ "serde", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cb-bench-micro" +version = "0.9.3" +dependencies = [ + "alloy", + "axum 0.8.4", + "cb-common", + "cb-pbs", + "cb-tests", + "criterion", + "tokio", +] + [[package]] name = "cb-bench-pbs" version = "0.9.3" @@ -1805,6 +1830,33 @@ dependencies = [ "windows-link 0.2.0", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.3.0" @@ -2060,6 +2112,42 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -3338,6 +3426,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3948,6 +4047,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -4595,6 +4705,12 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -4831,6 +4947,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.3" @@ -6589,6 +6733,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" diff --git a/Cargo.toml b/Cargo.toml index e2fcf44f..2a7e8255 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ bimap = { version = "0.6.3", features = ["serde"] } blsful = "^2.5" blst = "^0.3.15" bytes = "1.10.1" +criterion = { version = "0.5", features = ["html_reports"] } cb-cli = { path = "crates/cli" } cb-common = { path = "crates/common" } cb-metrics = { path = "crates/metrics" } diff --git a/benches/microbench/Cargo.toml b/benches/microbench/Cargo.toml new file mode 100644 index 00000000..185b51ee --- /dev/null +++ b/benches/microbench/Cargo.toml @@ -0,0 +1,19 @@ +[package] +edition.workspace = true +name = "cb-bench-micro" +rust-version.workspace = true +version.workspace = true + +[dependencies] +alloy.workspace = true +axum.workspace = true +cb-common.workspace = true +cb-pbs.workspace = true +cb-tests = { path = "../../tests" } +criterion.workspace = true +tokio.workspace = true + +[[bench]] +name = "get_header" +harness = false +path = "src/get_header.rs" diff --git a/benches/microbench/src/get_header.rs b/benches/microbench/src/get_header.rs new file mode 100644 index 00000000..738a5241 --- /dev/null +++ b/benches/microbench/src/get_header.rs @@ -0,0 +1,148 @@ +//! Criterion benchmarks for the `get_header` PBS flow. +//! +//! # What this measures +//! +//! The full `get_header` pipeline end-to-end: HTTP fan-out to N in-process mock +//! relays, response parsing, header validation, signature verification, and bid +//! selection. This is wall-clock timing — useful for local development feedback +//! and catching latency regressions across relay counts. +//! +//! Criterion runs each benchmark hundreds of times, applies statistical analysis, +//! and reports mean ± standard deviation. Results are saved to +//! `target/criterion/` as HTML reports (open `report/index.html`). +//! +//! # Running +//! +//! ```bash +//! # Run all benchmarks +//! cargo bench --package cb-bench-micro +//! +//! # Run a specific variant by filter +//! cargo bench --package cb-bench-micro -- 3_relays +//! +//! # Save a named baseline to compare against later +//! cargo bench --package cb-bench-micro -- --save-baseline main +//! +//! # Compare against a saved baseline +//! cargo bench --package cb-bench-micro -- --load-baseline main --save-baseline current +//! ``` +//! +//! # What is NOT measured +//! +//! - PBS HTTP server overhead (we call `get_header()` directly, bypassing axum routing) +//! - Mock relay startup time (servers are started once in setup, before timing begins) +//! - `HeaderMap` allocation (created once in setup, cloned cheaply per iteration) + +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use alloy::primitives::B256; +use axum::http::HeaderMap; +use cb_common::{pbs::GetHeaderParams, signer::random_secret, types::Chain}; +use cb_pbs::{PbsState, get_header}; +use cb_tests::{ + mock_relay::{MockRelayState, start_mock_relay_service}, + utils::{generate_mock_relay, get_pbs_static_config, to_pbs_config}, +}; +use criterion::{Criterion, black_box, criterion_group, criterion_main}; + +// Ports 19201–19205 are reserved for the microbenchmark mock relays. +const BASE_PORT: u16 = 19200; +const CHAIN: Chain = Chain::Hoodi; +const MAX_RELAYS: usize = 5; +const RELAY_COUNTS: [usize; 3] = [1, 3, MAX_RELAYS]; + +/// Benchmarks `get_header` across three relay-count variants. +/// +/// # Setup (runs once, not measured) +/// +/// All MAX_RELAYS mock relays are started up-front and shared across variants. +/// Each variant gets its own `PbsState` pointing to a different relay subset. +/// The mock relays are in-process axum servers on localhost. +/// +/// # Per-iteration (measured) +/// +/// Each call to `b.iter(|| ...)` runs `get_header()` once: +/// - Fans out HTTP requests to N mock relays concurrently +/// - Parses and validates each relay response (header data + BLS signature) +/// - Selects the highest-value bid +/// +/// `black_box(...)` prevents the compiler from optimizing away inputs or the +/// return value. Without it, the optimizer could see that the result is unused +/// and eliminate the call entirely, producing a meaningless zero measurement. +fn bench_get_header(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + + // Start all mock relays once and build one PbsState per relay-count variant. + // All relays share the same MockRelayState (and therefore the same signing key). + let (states, params) = rt.block_on(async { + let signer = random_secret(); + let pubkey = signer.public_key(); + let mock_state = Arc::new(MockRelayState::new(CHAIN, signer)); + + let relay_clients: Vec<_> = (0..MAX_RELAYS) + .map(|i| { + let port = BASE_PORT + 1 + i as u16; + tokio::spawn(start_mock_relay_service(mock_state.clone(), port)); + generate_mock_relay(port, pubkey.clone()).expect("relay client") + }) + .collect(); + + // Give all servers time to bind before benchmarking starts. + tokio::time::sleep(Duration::from_millis(200)).await; + + let params = GetHeaderParams { slot: 0, parent_hash: B256::ZERO, pubkey }; + + // Port 0 here is the port the PBS service itself would bind to for incoming + // validator requests. We call get_header() as a function directly, so no + // PBS server is started and this port is never used. The actual relay + // endpoints are carried inside the RelayClient objects (ports 19201–19205). + let states: Vec = RELAY_COUNTS + .iter() + .map(|&n| { + let config = + to_pbs_config(CHAIN, get_pbs_static_config(0), relay_clients[..n].to_vec()); + PbsState::new(config, PathBuf::new()) + }) + .collect(); + + (states, params) + }); + + // Empty HeaderMap matches what the PBS route handler receives for requests without + // custom headers. Created once here to avoid measuring its allocation per iteration. + let headers = HeaderMap::new(); + + // A BenchmarkGroup groups related functions so Criterion produces a single + // comparison table and chart. All variants share the name "get_header/". + let mut group = c.benchmark_group("get_header"); + + for (i, relay_count) in RELAY_COUNTS.iter().enumerate() { + let state = states[i].clone(); + let params = params.clone(); + let headers = headers.clone(); + + // bench_function registers one timing function. The closure receives a + // `Bencher` — calling `b.iter(|| ...)` is the measured hot loop. + // Everything outside `b.iter` is setup and not timed. + group.bench_function(format!("{relay_count}_relays"), |b| { + b.iter(|| { + // block_on drives the async future to completion on the shared + // runtime. get_header takes owned args, so we clone cheap types + // (Arc-backed state, stack-sized params) on each iteration. + rt.block_on(get_header( + black_box(params.clone()), + black_box(headers.clone()), + black_box(state.clone()), + )) + .expect("get_header failed") + }) + }); + } + + group.finish(); +} + +// criterion_group! registers bench_get_header as a benchmark group named "benches". +// criterion_main! generates the main() entry point that Criterion uses to run them. +criterion_group!(benches, bench_get_header); +criterion_main!(benches); diff --git a/justfile b/justfile index eb6dc69f..de70acfa 100644 --- a/justfile +++ b/justfile @@ -195,6 +195,45 @@ coverage-summary: coverage-clean: cargo llvm-cov clean --workspace +# ======================= +# === Microbenchmarks === +# ======================= +# +# Development Loop: +# 1. Run the current bench: just bench dev +# 2. Update code +# 3. Re-run the bench, logging the diff from the last run: just bench dev + +# Regression Test: +# 1. Save a baseline on the main branch: just bench main +# 2. On a PR branch, compare against it: just bench-compare main + +[doc(""" + Install tools required by the bench-* commands. + - cargo-criterion: CLI runner for Criterion benchmarks with richer output + - critcmp: baseline diffing tool used by bench-compare +""")] +bench-install-tools: + cargo install cargo-criterion critcmp + +[doc(""" + Run microbenchmarks and save results as a named baseline. Example: just bench main + + Compares against the last benchmark run of any kind, not the previous save of + this baseline name. Useful for tracking incremental changes since your last run. + For accurate baseline comparisons, use bench-compare instead. +""")] +bench baseline: + cargo bench --package cb-bench-micro -- --save-baseline {{baseline}} + +[doc(""" + Run microbenchmarks, save results as "current", then diff against a named baseline. + Example: just bench-compare main +""")] +bench-compare baseline: + cargo bench --package cb-bench-micro -- --save-baseline current + critcmp {{baseline}} current + # ================= # === Kurtosis === # ================= From e4a79a193cecb79dd72f2545b0885540a890a53e Mon Sep 17 00:00:00 2001 From: Joe Clapis Date: Thu, 12 Mar 2026 12:02:53 -0400 Subject: [PATCH 16/22] Add binary signing to GitHub releases (#433) Co-authored-by: Jason Vranek --- .github/workflows/release-gate.yml | 93 +++++++++++++++++++++ .github/workflows/release.yml | 129 +++++++++++++++++++++++++++-- RELEASE.md | 78 +++++++++++++++++ 3 files changed, 292 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/release-gate.yml create mode 100644 RELEASE.md diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 00000000..4fb5e30a --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,93 @@ +name: Release Gate + +on: + pull_request: + types: [closed] + branches: [main] + +jobs: + release-gate: + name: Tag and update release branches + runs-on: ubuntu-latest + # Only run when a release/ branch is merged (not just closed) + if: | + github.event.pull_request.merged == true && + startsWith(github.event.pull_request.head.ref, 'release/v') + + permissions: + contents: write + + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - uses: actions/checkout@v4 + with: + # Full history required for version comparison against existing tags + # and for the fast-forward push to stable/beta. + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Extract and validate version + id: version + run: | + BRANCH="${{ github.event.pull_request.head.ref }}" + NEW_VERSION="${BRANCH#release/}" + echo "new=${NEW_VERSION}" >> $GITHUB_OUTPUT + + # Determine if this is an RC + if echo "$NEW_VERSION" | grep -qE '\-rc[0-9]+$'; then + echo "is_rc=true" >> $GITHUB_OUTPUT + else + echo "is_rc=false" >> $GITHUB_OUTPUT + fi + + - name: Validate version is strictly increasing + run: | + NEW_VERSION="${{ steps.version.outputs.new }}" + + # Get the latest tag; if none exist yet, skip the comparison + LATEST_TAG=$(git tag --list 'v*' --sort=-version:refname | head -n1) + if [ -z "$LATEST_TAG" ]; then + echo "No existing tags found — skipping version comparison" + exit 0 + fi + + LATEST_VERSION="${LATEST_TAG#v}" + + python3 - <> $GITHUB_ENV + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -57,8 +61,8 @@ jobs: context: . push: false platforms: linux/amd64,linux/arm64 - cache-from: type=registry,ref=ghcr.io/commit-boost/buildcache:${{ matrix.target-crate}} - cache-to: type=registry,ref=ghcr.io/commit-boost/buildcache:${{ matrix.target-crate }},mode=max + cache-from: type=registry,ref=ghcr.io/${{ env.OWNER }}/buildcache:${{ matrix.target-crate}} + cache-to: type=registry,ref=ghcr.io/${{ env.OWNER }}/buildcache:${{ matrix.target-crate }},mode=max file: provisioning/build.Dockerfile outputs: type=local,dest=build build-args: | @@ -150,6 +154,31 @@ jobs: path: | ${{ matrix.name }}-${{ github.ref_name }}-darwin_${{ matrix.package-suffix }}.tar.gz + # Signs the binaries + sign-binaries: + needs: + - build-binaries-linux + - build-binaries-darwin + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: ./artifacts + pattern: "commit-boost*" + + - name: Sign binaries + uses: sigstore/gh-action-sigstore-python@v3.2.0 + with: + inputs: ./artifacts/**/*.tar.gz + + - name: Upload signatures + uses: actions/upload-artifact@v4 + with: + name: signatures-${{ github.ref_name }} + path: | + ./artifacts/**/*.sigstore.json + # Builds the PBS Docker image build-and-push-pbs-docker: needs: [build-binaries-linux] @@ -176,6 +205,9 @@ jobs: tar -xzf ./artifacts/commit-boost-${{ github.ref_name }}-linux_arm64/commit-boost-${{ github.ref_name }}-linux_arm64.tar.gz -C ./artifacts/bin mv ./artifacts/bin/commit-boost ./artifacts/bin/linux_arm64/commit-boost + - name: Set lowercase owner + run: echo "OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -198,8 +230,8 @@ jobs: build-args: | BINARIES_PATH=./artifacts/bin tags: | - ghcr.io/commit-boost/pbs:${{ github.ref_name }} - ${{ !contains(github.ref_name, 'rc') && 'ghcr.io/commit-boost/pbs:latest' || '' }} + ghcr.io/${{ env.OWNER }}/pbs:${{ github.ref_name }} + ${{ !contains(github.ref_name, 'rc') && format('ghcr.io/{0}/pbs:latest', env.OWNER) || '' }} file: provisioning/pbs.Dockerfile # Builds the Signer Docker image @@ -228,6 +260,9 @@ jobs: tar -xzf ./artifacts/commit-boost-${{ github.ref_name }}-linux_arm64/commit-boost-${{ github.ref_name }}-linux_arm64.tar.gz -C ./artifacts/bin mv ./artifacts/bin/commit-boost ./artifacts/bin/linux_arm64/commit-boost + - name: Set lowercase owner + run: echo "OWNER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -250,8 +285,8 @@ jobs: build-args: | BINARIES_PATH=./artifacts/bin tags: | - ghcr.io/commit-boost/signer:${{ github.ref_name }} - ${{ !contains(github.ref_name, 'rc') && 'ghcr.io/commit-boost/signer:latest' || '' }} + ghcr.io/${{ env.OWNER }}/signer:${{ github.ref_name }} + ${{ !contains(github.ref_name, 'rc') && format('ghcr.io/{0}/signer:latest', env.OWNER) || '' }} file: provisioning/signer.Dockerfile # Creates a draft release on GitHub with the binaries @@ -259,23 +294,101 @@ jobs: needs: - build-binaries-linux - build-binaries-darwin + - sign-binaries - build-and-push-pbs-docker - build-and-push-signer-docker runs-on: ubuntu-latest steps: - - name: Download artifacts + - name: Download binaries uses: actions/download-artifact@v4 with: path: ./artifacts pattern: "commit-boost*" + - name: Download signatures + uses: actions/download-artifact@v4 + with: + path: ./artifacts + pattern: "signatures-${{ github.ref_name }}*" + - name: Finalize Release uses: softprops/action-gh-release@v2 with: files: ./artifacts/**/* draft: true - prerelease: false + prerelease: ${{ contains(github.ref_name, '-rc') }} tag_name: ${{ github.ref_name }} name: ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Fast-forwards stable (full release) or beta (RC) to the new tag. + # Runs after all artifacts are built and the draft release is created, + # so stable/beta are never touched if any part of the pipeline fails. + fast-forward-branch: + needs: + - finalize-release + runs-on: ubuntu-latest + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Configure git + run: | + git config user.name "commit-boost-release-bot[bot]" + git config user.email "commit-boost-release-bot[bot]@users.noreply.github.com" + + - name: Fast-forward beta branch (RC releases) + if: contains(github.ref_name, '-rc') + run: | + git checkout beta + git merge --ff-only "${{ github.ref_name }}" + git push origin beta + + - name: Fast-forward stable branch (full releases) + if: "!contains(github.ref_name, '-rc')" + run: | + git checkout stable + git merge --ff-only "${{ github.ref_name }}" + git push origin stable + + # Deletes the tag if any job in the release pipeline fails. + # This keeps the tag and release artifacts in sync — a tag should only + # exist if the full pipeline completed successfully. + # stable/beta are never touched on failure since fast-forward-branch + # only runs after finalize-release succeeds. + # + # Note: if finalize-release specifically fails, a draft release may already + # exist on GitHub pointing at the now-deleted tag and will need manual cleanup. + cleanup-on-failure: + needs: + - build-binaries-linux + - build-binaries-darwin + - sign-binaries + - build-and-push-pbs-docker + - build-and-push-signer-docker + - finalize-release + - fast-forward-branch + runs-on: ubuntu-latest + if: failure() + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + + - name: Delete tag + run: git push origin --delete ${{ github.ref_name }} \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..846f39a7 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,78 @@ +# Releasing a new version of Commit-Boost + +## Process + +1. Cut a release candidate (RC) +2. Test the RC +3. Collect signoffs +4. Cut the full release + +## How it works + +Releases are fully automated once a release PR is merged into `main`. The branch name controls what CI does: + +| Branch name | Result | +| --- | --- | +| `release/vX.Y.Z-rcQ` | Creates RC tag, fast-forwards `beta`, builds and signs artifacts | +| `release/vX.Y.Z` | Creates release tag, fast-forwards `stable`, builds and signs artifacts | + +No human pushes tags or updates `stable`/`beta` directly, the CI handles everything after the PR merges. + +## Cutting a release candidate + +1. Create a branch named `release/vX.Y.Z-rc1`. For the first RC of a new version, bump the version in `Cargo.toml` and run `cargo check` to update `Cargo.lock`. Always update `CHANGELOG.md`. +2. Open a PR targeting `main`. Get two approvals and merge. +3. CI creates the tag, fast-forwards `beta`, builds and signs binaries, Docker images, and creates a draft release on GitHub. +4. Test the RC on testnets. For subsequent RCs (`-rc2`, etc.), open a new release PR with only a `CHANGELOG.md` update (`Cargo.toml` does not change between RCs). + +## Cutting the full release + +Once testing is complete and signoffs are collected: + +1. Create a branch named `release/vX.Y.Z` and update `CHANGELOG.md` with final release notes. +2. Open a PR targeting `main`. Get two approvals and merge. +3. CI creates the tag, fast-forwards `stable`, builds and signs artifacts, and creates a draft release. +4. Open the draft release on GitHub: + - Click **Generate release notes** and add a plain-language summary at the top + - Call out any breaking config changes explicitly + - Insert the [binary verification boilerplate text](#verifying-release-artifacts) + - Set as **latest release** (not pre-release) + - Publish +5. Update the community. + +## If the pipeline fails + +CI will automatically delete the tag if any build step fails. `stable` and `beta` are only updated after all artifacts are successfully built, they are never touched on a failed run. Fix the issue and open a new release PR. + +## Verifying release artifacts + +All binaries are signed using [Sigstore cosign](https://docs.sigstore.dev/about/overview/). You can verify any binary was built by the official Commit-Boost CI pipeline from this release's commit. + +Install cosign: https://docs.sigstore.dev/cosign/system_config/installation/ + +```bash +# Set the release version and your target architecture +# Architecture options: darwin_arm64, linux_arm64, linux_x86-64 +export VERSION=vX.Y.Z +export ARCH=linux_x86-64 + +# Download the binary tarball and its signature +curl -L \ + -o "commit-boost-$VERSION-$ARCH.tar.gz" \ + "https://github.com/Commit-Boost/commit-boost-client/releases/download/$VERSION/commit-boost-$VERSION-$ARCH.tar.gz" + +curl -L \ + -o "commit-boost-$VERSION-$ARCH.tar.gz.sigstore.json" \ + "https://github.com/Commit-Boost/commit-boost-client/releases/download/$VERSION/commit-boost-$VERSION-$ARCH.tar.gz.sigstore.json" + +# Verify the binary was signed by the official CI pipeline +cosign verify-blob \ + "commit-boost-$VERSION-$ARCH.tar.gz" \ + --bundle "commit-boost-$VERSION-$ARCH.tar.gz.sigstore.json" \ + --certificate-oidc-issuer="https://token.actions.githubusercontent.com" \ + --certificate-identity="https://github.com/Commit-Boost/commit-boost-client/.github/workflows/release.yml@refs/tags/$VERSION" +``` + +A successful verification prints `Verified OK`. If the binary was modified after being built by CI, this command will fail. + +The `.sigstore.json` bundle for each binary is attached to this release alongside the binary itself. From 145e18489aed56adf1aadfe9abaf1ceabf94a547 Mon Sep 17 00:00:00 2001 From: JasonVranek Date: Thu, 12 Mar 2026 16:09:23 +0000 Subject: [PATCH 17/22] Fix for code scanning alert no. 21: Code injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/release-gate.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 4fb5e30a..3d18697e 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -33,8 +33,10 @@ jobs: - name: Extract and validate version id: version + env: + BRANCH_REF: ${{ github.event.pull_request.head.ref }} run: | - BRANCH="${{ github.event.pull_request.head.ref }}" + BRANCH="$BRANCH_REF" NEW_VERSION="${BRANCH#release/}" echo "new=${NEW_VERSION}" >> $GITHUB_OUTPUT From 7035d41c119a734ecff34d670d45beaae49015e4 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 12 Mar 2026 09:35:12 -0700 Subject: [PATCH 18/22] prevent cmd injection and pin sigstore version --- .github/workflows/release-gate.yml | 7 ++++--- .github/workflows/release.yml | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 3d18697e..c5f1a1a4 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -48,9 +48,9 @@ jobs: fi - name: Validate version is strictly increasing + env: + NEW_VERSION: ${{ steps.version.outputs.new }} run: | - NEW_VERSION="${{ steps.version.outputs.new }}" - # Get the latest tag; if none exist yet, skip the comparison LATEST_TAG=$(git tag --list 'v*' --sort=-version:refname | head -n1) if [ -z "$LATEST_TAG" ]; then @@ -87,8 +87,9 @@ jobs: git config user.email "commit-boost-release-bot[bot]@users.noreply.github.com" - name: Create and push tag + env: + VERSION: ${{ steps.version.outputs.new }} run: | - VERSION="${{ steps.version.outputs.new }}" git tag "$VERSION" HEAD git push origin "$VERSION" # Branch fast-forwarding happens in release.yml after all artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d56de8d0..b27d3e42 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -168,7 +168,7 @@ jobs: pattern: "commit-boost*" - name: Sign binaries - uses: sigstore/gh-action-sigstore-python@v3.2.0 + uses: sigstore/gh-action-sigstore-python@a5caf349bc536fbef3668a10ed7f5cd309a4b53d #v3.2.0 with: inputs: ./artifacts/**/*.tar.gz @@ -391,4 +391,4 @@ jobs: token: ${{ steps.app-token.outputs.token }} - name: Delete tag - run: git push origin --delete ${{ github.ref_name }} \ No newline at end of file + run: git push origin --delete ${{ github.ref_name }} From 2ff0a879c605522475f299c3f91890d1292f02b6 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 12 Mar 2026 09:39:59 -0700 Subject: [PATCH 19/22] fmt code --- benches/microbench/src/get_header.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/benches/microbench/src/get_header.rs b/benches/microbench/src/get_header.rs index 738a5241..44eff329 100644 --- a/benches/microbench/src/get_header.rs +++ b/benches/microbench/src/get_header.rs @@ -7,8 +7,8 @@ //! selection. This is wall-clock timing — useful for local development feedback //! and catching latency regressions across relay counts. //! -//! Criterion runs each benchmark hundreds of times, applies statistical analysis, -//! and reports mean ± standard deviation. Results are saved to +//! Criterion runs each benchmark hundreds of times, applies statistical +//! analysis, and reports mean ± standard deviation. Results are saved to //! `target/criterion/` as HTML reports (open `report/index.html`). //! //! # Running @@ -29,9 +29,12 @@ //! //! # What is NOT measured //! -//! - PBS HTTP server overhead (we call `get_header()` directly, bypassing axum routing) -//! - Mock relay startup time (servers are started once in setup, before timing begins) -//! - `HeaderMap` allocation (created once in setup, cloned cheaply per iteration) +//! - PBS HTTP server overhead (we call `get_header()` directly, bypassing axum +//! routing) +//! - Mock relay startup time (servers are started once in setup, before timing +//! begins) +//! - `HeaderMap` allocation (created once in setup, cloned cheaply per +//! iteration) use std::{path::PathBuf, sync::Arc, time::Duration}; @@ -73,7 +76,8 @@ fn bench_get_header(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); // Start all mock relays once and build one PbsState per relay-count variant. - // All relays share the same MockRelayState (and therefore the same signing key). + // All relays share the same MockRelayState (and therefore the same signing + // key). let (states, params) = rt.block_on(async { let signer = random_secret(); let pubkey = signer.public_key(); @@ -108,8 +112,9 @@ fn bench_get_header(c: &mut Criterion) { (states, params) }); - // Empty HeaderMap matches what the PBS route handler receives for requests without - // custom headers. Created once here to avoid measuring its allocation per iteration. + // Empty HeaderMap matches what the PBS route handler receives for requests + // without custom headers. Created once here to avoid measuring its + // allocation per iteration. let headers = HeaderMap::new(); // A BenchmarkGroup groups related functions so Criterion produces a single @@ -142,7 +147,8 @@ fn bench_get_header(c: &mut Criterion) { group.finish(); } -// criterion_group! registers bench_get_header as a benchmark group named "benches". -// criterion_main! generates the main() entry point that Criterion uses to run them. +// criterion_group! registers bench_get_header as a benchmark group named +// "benches". criterion_main! generates the main() entry point that Criterion +// uses to run them. criterion_group!(benches, bench_get_header); criterion_main!(benches); From 31b5c09f73ceef639e6858bba195c83f815efe58 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 12 Mar 2026 18:13:17 -0700 Subject: [PATCH 20/22] CI to push release instead of draft to adhere to branch rules --- .github/workflows/release.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b27d3e42..6fe96c5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,3 +1,4 @@ + name: Draft Release on: @@ -289,7 +290,7 @@ jobs: ${{ !contains(github.ref_name, 'rc') && format('ghcr.io/{0}/signer:latest', env.OWNER) || '' }} file: provisioning/signer.Dockerfile - # Creates a draft release on GitHub with the binaries + # Creates a release on GitHub with the binaries finalize-release: needs: - build-binaries-linux @@ -299,6 +300,17 @@ jobs: - build-and-push-signer-docker runs-on: ubuntu-latest steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Download binaries uses: actions/download-artifact@v4 with: @@ -315,12 +327,13 @@ jobs: uses: softprops/action-gh-release@v2 with: files: ./artifacts/**/* - draft: true + draft: false prerelease: ${{ contains(github.ref_name, '-rc') }} tag_name: ${{ github.ref_name }} name: ${{ github.ref_name }} + generate_release_notes: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} # Fast-forwards stable (full release) or beta (RC) to the new tag. # Runs after all artifacts are built and the draft release is created, From 1fd6bed9b7a54cf73b06d3c5c1d0aba78d895bfc Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 16 Mar 2026 09:16:47 -0700 Subject: [PATCH 21/22] draft releases are removed from pipeline, so edit instructions --- RELEASE.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 846f39a7..4176edfb 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -31,13 +31,8 @@ Once testing is complete and signoffs are collected: 1. Create a branch named `release/vX.Y.Z` and update `CHANGELOG.md` with final release notes. 2. Open a PR targeting `main`. Get two approvals and merge. -3. CI creates the tag, fast-forwards `stable`, builds and signs artifacts, and creates a draft release. -4. Open the draft release on GitHub: - - Click **Generate release notes** and add a plain-language summary at the top - - Call out any breaking config changes explicitly - - Insert the [binary verification boilerplate text](#verifying-release-artifacts) - - Set as **latest release** (not pre-release) - - Publish +3. CI creates the tag, fast-forwards `stable`, builds and signs artifacts, and creates the release. +4. Verify the [binary was correctly signed](#verifying-release-artifacts). 5. Update the community. ## If the pipeline fails From c5582a41ea81f252b0cefffeafab77921fa0e11b Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 16 Mar 2026 12:52:23 -0700 Subject: [PATCH 22/22] Address review comments --- bin/commit-boost.rs | 2 -- crates/common/src/config/module.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/commit-boost.rs b/bin/commit-boost.rs index b8dc2261..e424d144 100644 --- a/bin/commit-boost.rs +++ b/bin/commit-boost.rs @@ -74,7 +74,6 @@ async fn run_pbs_service() -> Result<()> { maybe_err = server => { if let Err(err) = maybe_err { error!(%err, "PBS service unexpectedly stopped"); - eprintln!("PBS service unexpectedly stopped: {err}"); } }, _ = wait_for_signal() => { @@ -94,7 +93,6 @@ async fn run_signer_service() -> Result<()> { maybe_err = server => { if let Err(err) = maybe_err { error!(%err, "signing server unexpectedly stopped"); - eprintln!("signing server unexpectedly stopped: {err}"); } }, _ = wait_for_signal() => { diff --git a/crates/common/src/config/module.rs b/crates/common/src/config/module.rs index 05828f85..332560eb 100644 --- a/crates/common/src/config/module.rs +++ b/crates/common/src/config/module.rs @@ -14,7 +14,7 @@ use crate::{ types::{Chain, Jwt, ModuleId}, }; -#[derive(Debug, Deserialize, Serialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone, Copy)] pub enum ModuleKind { #[serde(alias = "commit")] Commit,