diff --git a/Cargo.lock b/Cargo.lock index 62a142cf..fd541d1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2581,6 +2581,7 @@ dependencies = [ "time", "tokio", "tokio-tar", + "tokio-tungstenite", "tokio-util", "tower-http", "tracing", @@ -4583,7 +4584,11 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", + "rustls 0.23.37", + "rustls-native-certs", + "rustls-pki-types", "tokio", + "tokio-rustls 0.26.4", "tungstenite", ] @@ -4782,6 +4787,8 @@ dependencies = [ "httparse", "log", "rand 0.9.2", + "rustls 0.23.37", + "rustls-pki-types", "sha1", "thiserror 2.0.18", "utf-8", diff --git a/netmito/Cargo.toml b/netmito/Cargo.toml index 4d8c246b..699b4e68 100644 --- a/netmito/Cargo.toml +++ b/netmito/Cargo.toml @@ -70,6 +70,7 @@ thiserror = "2.0.17" time = { version = "0.3.44", features = ["serde-human-readable"] } tokio = { workspace = true } tokio-tar = "0.3.1" +tokio-tungstenite = { version = "0.28", features = ["rustls-tls-native-roots"] } tokio-util = { version = "0.7.16", features = ["rt"] } tower-http = { version = "0.6.6", features = ["cors", "catch-panic"] } tracing = { workspace = true } diff --git a/netmito/src/agent.rs b/netmito/src/agent.rs new file mode 100644 index 00000000..8fedc7b2 --- /dev/null +++ b/netmito/src/agent.rs @@ -0,0 +1,1493 @@ +//! The agent: a process that accepts task suites from the coordinator and runs them. +//! +//! ```text +//! register ─▶ ws connect (unless --no-ws) ─┐ +//! heartbeat every N ───────────────────────┼─▶ main loop ─▶ (idle + work) ─▶ accept suite +//! idle poll every N (if configured) ───────┘ │ +//! ▼ +//! complete ◀─ cleanup hook ◀─ cleanup ◀─ tasks ◀─ start ◀─ provision hook +//! ``` +//! +//! The main loop only sends heartbeats and reads notifications. A [`SuiteRunner`] in a spawned +//! task runs the accepted suite. Thus the main loop and the suite do not block each other. + +use std::{ + collections::VecDeque, + path::PathBuf, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +use crossfire::{AsyncTx, MAsyncRx, MAsyncTx}; +use futures::{SinkExt as _, StreamExt as _}; +use reqwest::StatusCode; +use speedy::{Readable as _, Writable as _}; +use tokio::{ + sync::{watch, Notify}, + task::JoinHandle, +}; +use tokio_tungstenite::tungstenite::Message as WsMessage; +use tokio_util::sync::CancellationToken; +use tracing_subscriber::layer::SubscriberExt; +use url::Url; +use uuid::Uuid; + +use crate::{ + config::{AgentConfig, AgentConfigCli, TracingGuard}, + entity::{hook_tasks::HookType, state::AgentState}, + error::{self, Result}, + executor::{ + create_shared_dir, ensure_traversable_dir, reset_workspace, warn_unreachable_ancestors, + ExecKind, Executor, RESOURCE_FETCH_BUDGET, SINGLE_DOWNLOAD_BUDGET, UPLOAD_BUDGET, + }, + schema::*, + service::auth::{ + cred::get_user_credential, credential_guard::CredentialGuard, get_and_prompt_username, + }, +}; + +/// The number of task slots of a job. A slot is a working directory and an [`Executor`]. Two +/// tasks must not use the same slot at the same time, because the `result/` directory of the +/// process becomes its artifact. +/// +/// The value is `FixedWorkers.worker_count` of the suite. It is at least one, because a job with +/// no slot cannot run the tasks of the suite. +fn task_slots(suite: &TaskSuiteSpec) -> usize { + match suite.worker_schedule { + WorkerSchedulePlan::FixedWorkers { worker_count, .. } => (worker_count as usize).max(1), + } +} + +/// The progress of all the slots of a job, shared by the slots and the task buffer. +#[derive(Debug, Default)] +struct SlotProgress { + /// The number of tasks that ended, in any way. + completed: AtomicUsize, + /// Notified when a task ends. The coordinator gives no task to a job that holds all the tasks + /// that it can hold. Thus a free slot is a reason to fetch again, before the poll interval + /// ends. + freed: Notify, +} + +impl SlotProgress { + /// Record that a slot ended a task. + fn task_ended(&self) { + self.completed.fetch_add(1, Ordering::Release); + self.freed.notify_one(); + } + + fn completed(&self) -> usize { + self.completed.load(Ordering::Acquire) + } +} + +/// The interval at which a job with no work fetches tasks from its suite again. This is the +/// backup: the coordinator sends `TasksAvailable` for a new task in the suite. The poll is +/// necessary only when the notification cannot come: the WebSocket is down, or a slot of this job +/// submitted a downstream task. +const HOLD_POLL_INTERVAL: Duration = Duration::from_secs(1); + +pub struct MitoAgent { + coordinator_addr: Url, + token: String, + /// The highest notification id that the agent processed. Every heartbeat sends it. + notification_counter: u64, + /// The coordinator process that issued `notification_counter`. A different value means that + /// the coordinator restarted and the counter starts again. + coordinator_boot_id: Option, + state: AgentState, + assigned_suite_uuid: Option, + has_pending_suite: bool, + heartbeat_interval: Duration, + connect_retry_interval: Duration, + /// The interval at which an idle agent asks for a suite with no notification. `None` waits + /// for a notification. + idle_poll_interval: Option, + /// The time to wait before the agent opens a dropped notification WebSocket again. + ws_reconnect_interval: Duration, + /// Open no notification WebSocket. The heartbeat gives all notifications. + no_ws: bool, + /// The root of the working directories of this agent. Each job has one subdirectory. + cache_path: PathBuf, + http_client: reqwest::Client, + redis_client: Option, + /// Stops the job that runs now. `None` while the agent is idle. + job_token: Option, + /// Stops the job from claiming tasks, but lets the job end correctly. `None` while the agent + /// is idle. + drain_token: Option, + /// The job that runs now. Its result tells if more work is available for this agent. + current_run: Option>, + /// Stops the main loop. + shutdown_token: CancellationToken, + /// Set by the coordinator when it rejects the credential of the agent. + coordinator_force_exit: Arc, + /// Set by a graceful `Shutdown` notification or by Ctrl+C: finish the current job, then stop. + /// Do not accept a new suite. + stop_after_current: bool, + /// Increased when the coordinator notifies that the suite of the job has tasks. It is a counter + /// and not a flag: the receiver only sees that it changed, and a change during a fetch stays + /// visible for the next wait. + fetch_wake: watch::Sender, +} + +impl MitoAgent { + pub async fn main(cli: AgentConfigCli) { + // `set_default` and not `init`: this subscriber is only for the config and the + // registration. The real subscriber needs the agent id from the coordinator, thus `setup` + // installs it and drops this one. + let bootstrap = tracing::dispatcher::set_default(&tracing::Dispatch::new( + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "netmito=info".into()), + ) + .with( + tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true), + ), + )); + match AgentConfig::new(&cli) { + Ok(config) => match Self::setup(config, bootstrap).await { + Ok((mut agent, _guards)) => { + if let Err(e) = agent.run().await { + tracing::error!("Agent failed: {e}"); + } + agent.cleanup().await; + } + Err(e) => { + tracing::error!("{e}"); + } + }, + Err(e) => { + tracing::error!("{e}"); + } + } + } + + /// Register with the coordinator and prepare the working directory. + /// + /// `bootstrap` is the temporary subscriber of `main`. This function releases it when the real + /// subscriber is installed. A `DefaultGuard` is thread-local. This is safe because `main` runs + /// in the future of `block_on`, which always stays on the thread that calls it. + pub async fn setup( + mut config: AgentConfig, + bootstrap: tracing::subscriber::DefaultGuard, + ) -> Result<(Self, TracingGuard)> { + tracing::info!("Starting agent against {}", config.coordinator_addr); + tracing::info!( + "groups={:?} tags={:?} labels={:?}", + config.groups, + config.tags, + config.labels + ); + + let http_client = reqwest::Client::new(); + let mut credential_guard = CredentialGuard::new( + config + .credential_path + .as_ref() + .map(|credential_path| credential_path.relative()), + &config.coordinator_addr, + ) + .await; + let username = match &config.user { + Some(name) => name.to_string(), + None => get_and_prompt_username(None, "Please input username")?, + }; + // Like the worker: a restart of the agent must not change the auth signature of the user. + // A new signature makes all the tokens issued before it invalid. + let (_, user_credential) = get_user_credential( + &mut credential_guard, + &http_client, + config.coordinator_addr.clone(), + username, + config.password.take(), + false, + ) + .await?; + + let machine_code = Self::resolve_machine_code(config.machine_code.take()); + tracing::info!("Machine code: {machine_code}"); + + let mut register_url = config.coordinator_addr.clone(); + register_url.set_path("agents"); + let req = RegisterAgentReq { + tags: config.tags.clone(), + labels: config.labels.clone(), + admin_group: config.admin_group.clone(), + groups: config.groups.clone(), + lifetime: config.lifetime, + machine_code, + metadata: Some(AgentMetadata { + version: env!("CARGO_PKG_VERSION").to_string(), + long_version: env!("CARGO_PKG_VERSION").to_string(), + }), + }; + let resp = http_client + .post(register_url.as_str()) + .bearer_auth(&user_credential) + .json(&req) + .send() + .await + .map_err(|e| { + if e.is_request() && e.is_connect() { + error::RequestError::ConnectionError(config.coordinator_addr.to_string()) + } else { + e.into() + } + })?; + let register: RegisterAgentResp = parse_json(resp, "register agent").await?; + + let mut cache_path = + dirs::cache_dir().ok_or(error::Error::Custom("Cache dir not found".to_string()))?; + // Make one level at a time, and make each level traversable. `create_dir_all` applies the + // umask to the levels that it creates, and one `0700` level hides the working directory + // from a task that changed user. + warn_unreachable_ancestors(&cache_path).await; + cache_path.push("mitosis"); + ensure_traversable_dir(&cache_path).await?; + cache_path.push("agent"); + ensure_traversable_dir(&cache_path).await?; + cache_path.push(register.agent_uuid.to_string()); + ensure_traversable_dir(&cache_path).await?; + + let guards = config.setup_tracing_subscriber::<&Uuid, _>(®ister.agent_uuid)?; + // The subscriber of the agent is installed now. Remove the temporary one. + drop(bootstrap); + tracing::info!( + "Registered as agent {} ({})", + register.agent_uuid, + if register.reused { + "re-adopted this machine's existing agent" + } else { + "new agent" + } + ); + tracing::info!("Working directory: {}", cache_path.display()); + + let redis_client = if config.skip_redis { + None + } else { + register.redis_url.and_then(|url| { + redis::Client::open(url) + .inspect_err(|e| tracing::warn!("Agent cannot setup redis conn: {}", e)) + .ok() + }) + }; + + let mut coordinator_addr = config.coordinator_addr; + coordinator_addr.set_path(""); + + Ok(( + MitoAgent { + coordinator_addr, + token: register.token, + notification_counter: register.notification_counter, + coordinator_boot_id: None, + state: AgentState::Idle, + assigned_suite_uuid: None, + // The coordinator cannot notify an agent before it exists. Thus the registration + // is also a reason to look for work. + has_pending_suite: true, + heartbeat_interval: config.heartbeat_interval, + connect_retry_interval: config.connect_retry_interval, + idle_poll_interval: config.idle_poll_interval, + ws_reconnect_interval: config.ws_reconnect_interval, + no_ws: config.no_ws, + cache_path, + http_client, + redis_client, + job_token: None, + drain_token: None, + current_run: None, + shutdown_token: CancellationToken::new(), + coordinator_force_exit: Arc::new(AtomicBool::new(false)), + stop_after_current: false, + fetch_wake: watch::Sender::new(0), + }, + guards, + )) + } + + /// Find a stable machine code for this host. + /// + /// The order: the config value, the cached value in the mitosis config directory, + /// `/etc/machine-id`, a new UUID. The agent caches the code that it finds (but not a config + /// value). Thus the code stays the same after a restart, also on a host with no + /// `/etc/machine-id` (a container, a host that is not Linux). The coordinator finds the agent + /// row by this code, thus a machine that comes back with a different code leaves its old row. + fn resolve_machine_code(explicit: Option) -> String { + if let Some(code) = explicit + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + { + return code; + } + + let cache_path = dirs::config_dir().map(|mut p| { + p.push("mitosis"); + p.push("machine-id"); + p + }); + if let Some(ref path) = cache_path { + if let Ok(cached) = std::fs::read_to_string(path) { + let cached = cached.trim().to_string(); + if !cached.is_empty() { + return cached; + } + } + } + + let code = std::fs::read_to_string("/etc/machine-id") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| { + tracing::info!("No /etc/machine-id available; generating a machine code"); + Uuid::new_v4().simple().to_string() + }); + + match cache_path { + Some(path) => { + let res = path + .parent() + .map(std::fs::create_dir_all) + .unwrap_or(Ok(())) + .and_then(|_| std::fs::write(&path, &code)); + if let Err(e) = res { + tracing::warn!( + "Failed to cache the machine code at {}: {e}; a new one may be generated next start", + path.display() + ); + } + } + None => tracing::warn!( + "No config directory to cache the machine code in; a new one may be generated next start" + ), + } + + code + } + + fn api_url(&self, path: &str) -> Url { + let mut url = self.coordinator_addr.clone(); + url.set_path(path); + url + } + + pub async fn run(&mut self) -> Result<()> { + let cancel_token = self.shutdown_token.clone(); + + // Cancelled when the first Ctrl+C comes. + let signal_token = CancellationToken::new(); + tokio::spawn({ + let shutdown_token = cancel_token.clone(); + let signal_token = signal_token.clone(); + async move { + crate::signal::shutdown_signal(shutdown_token).await; + signal_token.cancel(); + } + }); + + // With `no_ws`, the agent opens no socket and the branch that reads it is off. The + // channel exists in all cases, because `select!` evaluates the expression of a branch + // also when its guard is false. + let ws_enabled = !self.no_ws; + let (ws_tx, ws_rx) = crossfire::spsc::bounded_async::(32); + let ws_handle = if ws_enabled { + Some(self.spawn_websocket_client(ws_tx, cancel_token.clone())) + } else { + tracing::info!("WebSocket disabled; Notifications rely purely on heartbeat"); + None + }; + + let mut heartbeat_timer = tokio::time::interval(self.heartbeat_interval); + heartbeat_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + let (idle_poll_enabled, mut idle_poll_timer) = match self.idle_poll_interval { + Some(interval) => { + tracing::info!( + "Idle agent will poll for a suite every {}s", + interval.as_secs() + ); + let mut timer = tokio::time::interval(interval); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + (true, timer) + } + None => { + // The branch of this timer is off when `idle_poll_enabled` is false. Thus this + // timer never fires. + let mut timer = tokio::time::interval(self.heartbeat_interval); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + (false, timer) + } + }; + + tracing::info!("Agent entering its main loop"); + loop { + tokio::select! { + biased; + _ = cancel_token.cancelled() => { + tracing::info!("Shutdown signalled, leaving the main loop"); + // Stop the job, thus its reports stop. The coordinator reclaims its tasks + // when the heartbeat of the agent stops. + if let Some(token) = &self.job_token { + token.cancel(); + } + break; + } + + // Off after a graceful stop starts. Thus this branch does not fire again on each + // pass, and a second Ctrl+C goes to the shared handler, which stops the process. + _ = signal_token.cancelled(), if !self.stop_after_current => { + tracing::info!( + "Signal received; finishing the current job before exiting \ + (Ctrl+C again to exit now)" + ); + self.begin_graceful_shutdown(); + } + + Ok(event) = ws_rx.recv(), if ws_enabled => { + self.notification_counter = self.notification_counter.max(event.id); + self.handle_notification(event.event); + } + + _ = heartbeat_timer.tick() => { + if let Err(e) = self.send_heartbeat().await { + tracing::error!("Heartbeat failed: {e}"); + } + } + + _ = idle_poll_timer.tick(), if idle_poll_enabled => { + // The `advance` below then asks for a suite. A busy agent ignores the flag. + if self.state == AgentState::Idle && !self.stop_after_current { + self.has_pending_suite = true; + } + } + } + + if let Err(e) = self.advance().await { + tracing::error!("Agent state handling failed: {e}"); + } + if self.stop_after_current && self.current_run.is_none() { + tracing::info!("Nothing left to run and a stop was requested; exiting"); + cancel_token.cancel(); + break; + } + } + + drop(ws_rx); + // Let the job end, thus its last reports arrive. + if let Some(handle) = self.current_run.take() { + let _ = handle.await; + } + if let Some(handle) = ws_handle { + let _ = handle.await; + } + if self.coordinator_force_exit.load(Ordering::Acquire) { + tracing::info!("Agent exits because the coordinator rejected its credential"); + } else { + tracing::info!("Agent stopped"); + } + Ok(()) + } + + /// Tell the coordinator that this agent stops. Then the coordinator does not wait for the + /// heartbeat timeout. + /// + /// The timeout is short, and a failure is only logged: a coordinator that does not answer must + /// not keep the process open, and its heartbeat check retires the agent later. + pub async fn cleanup(&self) { + if let Err(e) = self + .http_client + .delete(self.api_url("agents").as_str()) + .bearer_auth(&self.token) + .timeout(Duration::from_secs(3)) + .send() + .await + { + tracing::warn!("Failed to tell the coordinator we are exiting: {e}"); + } + } + + fn spawn_websocket_client( + &self, + mut notification_tx: AsyncTx, + cancel_token: CancellationToken, + ) -> JoinHandle<()> { + let mut url = self.coordinator_addr.clone(); + let _ = url.set_scheme(if url.scheme() == "https" { "wss" } else { "ws" }); + url.set_path("ws/agents"); + let ws_url = url.to_string(); + let token = self.token.clone(); + let reconnect_interval = self.ws_reconnect_interval; + + tokio::spawn(async move { + while !cancel_token.is_cancelled() { + tracing::debug!("Connecting to {ws_url}"); + match Self::websocket_session(&ws_url, &token, &mut notification_tx, &cancel_token) + .await + { + Ok(()) => tracing::debug!("Agent WebSocket closed"), + // The heartbeat gives the same events, thus a failure here is not fatal. + Err(e) => tracing::warn!("Agent WebSocket error: {e}"), + } + tokio::select! { + _ = cancel_token.cancelled() => break, + _ = tokio::time::sleep(reconnect_interval) => {} + } + } + tracing::debug!("Agent WebSocket task stopped"); + }) + } + + /// One WebSocket session: read the notifications, give each one to the main loop, then + /// acknowledge it. Returns when the socket closes or the agent stops. + async fn websocket_session( + ws_url: &str, + token: &str, + notification_tx: &mut AsyncTx, + cancel_token: &CancellationToken, + ) -> Result<()> { + let host = Url::parse(ws_url) + .ok() + .and_then(|u| { + u.host_str().map(|h| match u.port() { + Some(port) => format!("{h}:{port}"), + None => h.to_string(), + }) + }) + .unwrap_or_default(); + let request = tokio_tungstenite::tungstenite::http::Request::builder() + .uri(ws_url) + .header("Host", host) + .header("Authorization", format!("Bearer {token}")) + .header("Sec-WebSocket-Version", "13") + .header("Connection", "Upgrade") + .header("Upgrade", "websocket") + .header( + "Sec-WebSocket-Key", + tokio_tungstenite::tungstenite::handshake::client::generate_key(), + ) + .body(()) + .map_err(|e| error::Error::Custom(format!("Invalid WebSocket request: {e}")))?; + + let (ws_stream, _) = tokio_tungstenite::connect_async(request) + .await + .map_err(|e| error::Error::Custom(format!("WebSocket connect failed: {e}")))?; + tracing::info!("Agent WebSocket connected"); + + let (mut ws_write, mut ws_read) = ws_stream.split(); + loop { + let msg = tokio::select! { + _ = cancel_token.cancelled() => return Ok(()), + msg = ws_read.next() => msg, + }; + let Some(msg) = msg else { return Ok(()) }; + match msg { + Ok(WsMessage::Binary(bytes)) => { + let event = match WsNotificationEvent::read_from_buffer(&bytes) { + Ok(event) => event, + Err(e) => { + tracing::warn!("Unparseable notification: {e}"); + continue; + } + }; + let id = event.id; + let send_result = tokio::select! { + biased; + _ = cancel_token.cancelled() => return Ok(()), + result = notification_tx.send(event) => result, + }; + if send_result.is_err() { + return Ok(()); + } + // Acknowledge only after the main loop has the event. Thus the coordinator + // keeps the event for a replay until the agent has it. + let ack = AgentWsMessage::Ack { + notification_id: id, + }; + if let Ok(payload) = ack.write_to_vec() { + let _ = ws_write.send(WsMessage::Binary(payload.into())).await; + } + } + Ok(WsMessage::Ping(_)) | Ok(WsMessage::Pong(_)) | Ok(WsMessage::Frame(_)) => {} + Ok(WsMessage::Text(text)) => { + tracing::debug!(%text, "Ignoring an unexpected text frame") + } + Ok(WsMessage::Close(frame)) => { + tracing::debug!(?frame, "Coordinator closed the WebSocket"); + return Ok(()); + } + Err(e) => return Err(error::Error::Custom(format!("WebSocket error: {e}"))), + } + } + } + + /// Start a graceful stop: the tasks that run now finish and the cleanup hook runs, but the + /// job claims no more tasks and the agent accepts no new suite. An idle agent leaves the main + /// loop on the next pass. + fn begin_graceful_shutdown(&mut self) { + self.stop_after_current = true; + self.has_pending_suite = false; + if let Some(token) = &self.drain_token { + token.cancel(); + } + } + + fn handle_notification(&mut self, notification: AgentNotification) { + match notification { + AgentNotification::SuiteAvailable => { + tracing::debug!("A suite is available"); + self.has_pending_suite = true; + } + AgentNotification::TasksAvailable { suite_uuid } => { + // For the job that runs now, not for the idle loop: the coordinator keeps tasks + // for this job and asks it to fetch now, before its poll interval ends. + if self.assigned_suite_uuid == Some(suite_uuid) { + tracing::debug!(%suite_uuid, "Tasks are waiting for this job"); + self.fetch_wake.send_modify(|n| *n = n.wrapping_add(1)); + } else { + tracing::debug!(%suite_uuid, "Ignoring a task offer for a suite we left"); + } + } + AgentNotification::PreemptSuite { + new_suite_uuid, + current_suite_uuid, + .. + } => { + // The coordinator does not send this yet. The agent handles it now, thus only the + // coordinator must change to start preemption. + if self.assigned_suite_uuid == Some(current_suite_uuid) { + tracing::info!("Preempting suite {current_suite_uuid} for {new_suite_uuid}"); + if let Some(token) = &self.job_token { + token.cancel(); + } + self.has_pending_suite = true; + } + } + AgentNotification::SuiteCancelled { + suite_uuid, + reason, + graceful, + } => { + if self.assigned_suite_uuid == Some(suite_uuid) { + tracing::warn!( + "Suite {suite_uuid} was cancelled (graceful={graceful}): {reason}" + ); + // A graceful cancel only drains. The cleanup hook runs under the job token, + // thus a cancel of the job token also stops the cleanup hook. + let token = if graceful { + &self.drain_token + } else { + &self.job_token + }; + if let Some(token) = token { + token.cancel(); + } + } + } + AgentNotification::TasksCancelled { task_uuids } => { + // The agent keeps no record for this: a report for a task that is gone gets 404, + // and the slot continues with the next task. + tracing::debug!(count = task_uuids.len(), "Tasks cancelled"); + } + AgentNotification::StopJob { + suite_uuid, + graceful, + } => { + // Compare the suite: a stop that comes after the agent moved to a different suite + // must not stop the new job. + if self.assigned_suite_uuid == Some(suite_uuid) { + tracing::info!( + "A user asked this agent to stop its job on suite {suite_uuid} \ + (graceful={graceful})" + ); + // Do not set `stop_after_current`: after a stop, the agent picks a suite + // again. This is how the agent gets to work with a higher priority. + let token = if graceful { + &self.drain_token + } else { + &self.job_token + }; + if let Some(token) = token { + token.cancel(); + } + } + } + AgentNotification::Shutdown { graceful } => { + tracing::warn!("Coordinator asked this agent to shut down (graceful={graceful})"); + if graceful { + self.begin_graceful_shutdown(); + } else { + if let Some(token) = &self.job_token { + token.cancel(); + } + self.shutdown_token.cancel(); + } + } + AgentNotification::CounterSync { counter, boot_id } => { + if self.coordinator_boot_id != Some(boot_id) { + if let Some(previous) = self.coordinator_boot_id { + tracing::warn!("Coordinator restarted ({previous} → {boot_id})"); + } + self.coordinator_boot_id = Some(boot_id); + self.notification_counter = counter; + } else if counter > self.notification_counter { + self.notification_counter = counter; + } + } + }; + } + + async fn send_heartbeat(&mut self) -> Result<()> { + let req = AgentHeartbeatReq { + state: self.state, + last_notification_id: self.notification_counter, + }; + let resp = self + .http_client + .post(self.api_url("agents/heartbeat").as_str()) + .bearer_auth(&self.token) + .json(&req) + .send() + .await + .map_err(error::map_reqwest_err)?; + if resp.status() == StatusCode::UNAUTHORIZED { + // The coordinator retired this agent: the heartbeat timed out, or a user stopped the + // agent with force. The coordinator accepts no more calls with this token. Only a new + // registration can continue, and the supervisor of the process does that. Thus stop + // as a forced shutdown does. + tracing::warn!("The coordinator no longer recognises this agent; exiting"); + self.coordinator_force_exit.store(true, Ordering::Release); + if let Some(token) = &self.job_token { + token.cancel(); + } + self.shutdown_token.cancel(); + return Ok(()); + } + let resp: AgentHeartbeatResp = parse_json(resp, "heartbeat").await?; + + // The notifications that the WebSocket did not give come here. + for event in resp.notifications { + self.notification_counter = self.notification_counter.max(event.id); + self.handle_notification(event.event); + } + Ok(()) + } + + async fn advance(&mut self) -> Result<()> { + match self.state { + AgentState::Idle => { + if self.stop_after_current { + return Ok(()); + } + // Ask for a suite only after a signal: a notification, the registration, the end + // of a job with more work, or the idle poll. + if !self.has_pending_suite { + return Ok(()); + } + // One request for each signal. If no suite is available, the agent stays idle + // until the next signal and does not ask again in a loop. + self.has_pending_suite = false; + + let Some((suite, job, job_id, hooks)) = self.fetch_suite().await? else { + return Ok(()); + }; + tracing::info!("Accepted suite {} as job {job_id}", suite.uuid); + + self.assigned_suite_uuid = Some(suite.uuid); + let job_token = CancellationToken::new(); + self.job_token = Some(job_token.clone()); + let drain_token = CancellationToken::new(); + self.drain_token = Some(drain_token.clone()); + let runner = SuiteRunner { + http_client: self.http_client.clone(), + token: self.token.clone(), + coordinator_addr: self.coordinator_addr.clone(), + connect_retry_interval: self.connect_retry_interval, + redis_client: self.redis_client.clone(), + job_id, + hooks: Arc::new(hooks), + cache_path: self.cache_path.join("job"), + job_token, + shutdown_token: self.shutdown_token.clone(), + coordinator_force_exit: self.coordinator_force_exit.clone(), + drain_token, + fetch_wake: self.fetch_wake.subscribe(), + }; + self.current_run = Some(tokio::spawn(runner.run(suite, job))); + self.state = AgentState::Executing; + } + AgentState::Executing => { + // Do not wait for the job here, thus the heartbeats continue. + let finished = self + .current_run + .as_ref() + .map(JoinHandle::is_finished) + .unwrap_or(true); + if finished { + if let Some(handle) = self.current_run.take() { + match handle.await { + // The answer to `Complete` says that more work waits for this agent. + Ok(true) => self.has_pending_suite = true, + Ok(false) => {} + Err(e) => tracing::error!("The suite runner task failed: {e}"), + } + } + self.assigned_suite_uuid = None; + self.job_token = None; + self.drain_token = None; + self.state = AgentState::Idle; + tracing::info!("Agent is idle again"); + } + } + // The coordinator owns these states. The agent only reports `Idle` and `Executing`; + // the job reports `Start` and `EnterCleanup` mark its other phases. + AgentState::Provisioning | AgentState::Cleaning | AgentState::Offline => {} + } + Ok(()) + } + + /// Ask for a suite and accept it in one call. `None` means that no suite is available. + async fn fetch_suite(&self) -> Result)>> { + let resp = self + .http_client + .post(self.api_url("agents/suite").as_str()) + .bearer_auth(&self.token) + .json(&AcceptSuiteReq { suite_uuid: None }) + .send() + .await + .map_err(error::map_reqwest_err)?; + let resp: AcceptSuiteResp = parse_json(resp, "accept suite").await?; + if !resp.accepted { + tracing::debug!("No suite accepted: {}", resp.reason.unwrap_or_default()); + return Ok(None); + } + let Some(suite) = resp.suite else { + return Err(error::Error::Custom( + "the coordinator accepted a suite but sent no spec".to_string(), + )); + }; + let hooks = resp.hooks; + Ok(resp + .job + .zip(resp.job_id) + .map(|(job, id)| (suite, job, id, hooks))) + } +} + +/// Runs one accepted suite from the provision hook to a terminal job state. +#[derive(Clone)] +struct SuiteRunner { + http_client: reqwest::Client, + token: String, + coordinator_addr: Url, + connect_retry_interval: Duration, + redis_client: Option, + /// The number of the job in its suite. Use it in logs: `suites jobs --job N` takes + /// it. The `job` handle in the calls below is `suite_agent_jobs.id`, an internal key that the + /// user does not see. + job_id: i32, + /// The hook rows of the job, from the accept. + hooks: Arc>, + /// The working directory of this job. + cache_path: PathBuf, + /// Cancelled when the suite is cancelled or preempted, or when the agent stops. All that the + /// job runs (tasks, hooks) runs under it. + job_token: CancellationToken, + /// The shutdown of the agent. Only this token stops the retries of a job report, because the + /// agent must report a cancelled job too. + shutdown_token: CancellationToken, + coordinator_force_exit: Arc, + /// Cancelled to stop the claims of tasks, but not the end of the job: the running tasks + /// finish and commit, then the cleanup hook runs under `job_token`. Thus the two tokens are + /// different. The job drops the tasks that it claimed but did not start, and the coordinator + /// reclaims them at `Complete`. + drain_token: CancellationToken, + /// Changes when the coordinator notifies that tasks of this suite wait for this job. With it, the + /// job does not wait for the next poll. The poll stays as the backup for a dropped socket. + fetch_wake: watch::Receiver, +} + +impl SuiteRunner { + fn api_url(&self, path: &str) -> Url { + let mut url = self.coordinator_addr.clone(); + url.set_path(path); + url + } + + /// Each phase logs an error and continues, because only the agent can move the job to a + /// terminal state and free itself. + /// + /// Returns true if the answer to `Complete` says that more work waits for this agent. + async fn run(self, suite: TaskSuiteSpec, job: i64) -> bool { + let suite_uuid = suite.uuid; + let mut failure: Option = None; + + let provisioned = match self.prepare_workspace().await { + Ok(()) => self.run_hook(&suite, HookType::Provision).await, + Err(e) => Err(e), + }; + if let Err(e) = provisioned { + tracing::error!("Provisioning failed for suite {suite_uuid}: {e}"); + failure = Some(JobFailureReason { + kind: JobFailureKind::ProvisionFailed, + message: e.to_string(), + }); + } + + if let Err(e) = self.report_job(job, JobReportOp::Start).await { + tracing::error!("Failed to start suite {suite_uuid}: {e}"); + } + + let background_stop_token = CancellationToken::new(); + let mut background = None; + + // A stopped job runs no task and no background hook. + if failure.is_none() && !self.job_token.is_cancelled() { + background = self.spawn_background_hook(&suite, background_stop_token.clone()); + if let Err(e) = self.execute_tasks(&suite, job).await { + tracing::error!("Task execution failed for suite {suite_uuid}: {e}"); + failure = Some(JobFailureReason { + kind: JobFailureKind::ExecutionError, + message: e.to_string(), + }); + } + } + + if let Some(handle) = background { + // A background hook must run until the tasks end. If it stops first, the job failed. + // The hook cannot know when the job has no more tasks, thus the agent stops it. The + // executor then reports the hook with its output. + let exited_early = handle.is_finished(); + background_stop_token.cancel(); + if let Err(e) = handle.await { + tracing::error!("The background hook task failed for suite {suite_uuid}: {e}"); + } + if exited_early { + failure.get_or_insert(JobFailureReason { + kind: JobFailureKind::BackgroundExited, + message: "the background hook exited before the tasks drained".to_string(), + }); + } + } + + if let Err(e) = self.report_job(job, JobReportOp::EnterCleanup).await { + tracing::error!("Failed to enter cleanup for suite {suite_uuid}: {e}"); + } + + if let Err(e) = self.run_hook(&suite, HookType::Cleanup).await { + tracing::error!("Cleanup hook failed for suite {suite_uuid}: {e}"); + failure.get_or_insert(JobFailureReason { + kind: JobFailureKind::CleanupFailed, + message: e.to_string(), + }); + } + + let outcome = match failure { + None => SuiteJobOutcome::Completed, + Some(reason) => SuiteJobOutcome::Failed { reason }, + }; + let next_available = match self + .report_job(job, JobReportOp::Complete { outcome }) + .await + { + Ok(Some(resp)) => { + let next_available = resp.next_suite_available.unwrap_or(false); + tracing::info!( + "Finished suite {suite_uuid} (job {}); more work waiting: {next_available}", + self.job_id + ); + next_available + } + Ok(None) => false, + // A `Complete` that did not arrive tells nothing about the next work. + Err(e) => { + tracing::error!("Failed to complete suite {suite_uuid}: {e}"); + false + } + }; + + // All the outputs are uploaded now. + let _ = tokio::fs::remove_dir_all(&self.cache_path).await; + next_available + } + + /// The directory that the provision hook, the tasks and the cleanup hook all get as + /// `MITO_SUITE_SHARED`. + fn shared_path(&self) -> PathBuf { + self.cache_path.join("share") + } + + /// Create the file tree of a job. The processes only go through the job root. They get + /// `share/` as `MITO_SUITE_SHARED`, thus it is open to the user that they run as. + async fn prepare_workspace(&self) -> Result<()> { + let _ = tokio::fs::remove_dir_all(&self.cache_path).await; + ensure_traversable_dir(&self.cache_path).await?; + create_shared_dir(&self.shared_path()).await?; + Ok(()) + } + + /// Make an executor for one slot or one hook of this job, with its own working directory. + async fn executor( + &self, + cache_path: PathBuf, + cancel_token: CancellationToken, + stop_token: CancellationToken, + ) -> Executor { + let task_redis_conn = if let Some(ref client) = self.redis_client { + client + .get_multiplexed_tokio_connection() + .await + .inspect_err(|e| tracing::warn!("{}", e)) + .ok() + } else { + None + }; + let task_redis_pubsub = if let Some(ref client) = self.redis_client { + client + .get_async_pubsub() + .await + .inspect_err(|e| tracing::warn!("{}", e)) + .ok() + } else { + None + }; + Executor { + http_client: self.http_client.clone(), + task_credential: self.token.clone(), + http_url: self.coordinator_addr.clone(), + task_cancel_token: cancel_token, + task_stop_token: stop_token, + runner_cancel_token: self.shutdown_token.clone(), + coordinator_force_exit: self.coordinator_force_exit.clone(), + polling_interval: self.connect_retry_interval, + task_cache_path: cache_path, + extra_envs: vec![( + "MITO_SUITE_SHARED".to_string(), + self.shared_path().to_string_lossy().into_owned(), + )], + task_redis_conn, + task_redis_pubsub, + resource_fetch_budget: RESOURCE_FETCH_BUDGET, + single_download_budget: SINGLE_DOWNLOAD_BUDGET, + upload_budget: UPLOAD_BUDGET, + } + } + + /// Claim the tasks of the suite into a buffer and run them in `slots` slots. + /// + /// This function keeps the buffer: it does all the claims, all the fetches of a job that + /// waits for work, and the decision to end the job. The slots only take the tasks that it + /// gives them. Thus this work is not done once for each slot. + /// + /// The coordinator decides how many tasks the job can claim at a time: it knows the schedule + /// of the suite and the tasks that this job holds. The channel holds one task for each slot, + /// thus `send` waits when each slot has its next task. `staged` holds the remaining tasks of a + /// batch. + async fn execute_tasks(&self, suite: &TaskSuiteSpec, job: i64) -> Result<()> { + let slots = task_slots(suite); + tracing::info!( + "Running the tasks of suite {} across {slots} slots", + suite.uuid, + ); + + let (task_tx, task_rx) = crossfire::mpmc::bounded_async::(slots); + // Compared with the count of tasks that the buffer gave. Equal counts mean that all the + // tasks ended. A check of idle slots cannot tell this, because a slot can take a task + // during the check. + let progress = Arc::new(SlotProgress::default()); + + let mut running = tokio::task::JoinSet::new(); + for slot in 0..slots { + let runner = self.clone(); + let task_rx = task_rx.clone(); + let progress = progress.clone(); + running.spawn(async move { runner.run_slot(job, slot, task_rx, progress).await }); + } + // Only the slots keep a receiver. + drop(task_rx); + + let filled = self.fill_buffer(suite.uuid, task_tx, progress).await; + + while let Some(joined) = running.join_next().await { + if let Err(e) = joined { + tracing::error!("A task slot of suite {} panicked: {e}", suite.uuid); + } + } + filled + } + + /// Keep tasks in the buffer until the suite has no more tasks for this job. + /// + /// Returns when the job stops claiming: the suite has no more tasks, the job was cancelled, + /// or the job drains. The job drops the tasks that are still staged then, and `Complete` + /// gives them back to `Ready`. + async fn fill_buffer( + &self, + suite_uuid: Uuid, + task_tx: MAsyncTx, + progress: Arc, + ) -> Result<()> { + let mut staged: VecDeque = VecDeque::new(); + // Cloned once for the full loop, not for each wait. Thus a change that comes during a fetch + // ends the next wait. + let mut wake = self.fetch_wake.clone(); + // The number of tasks given to the slots. + let mut pushed: usize = 0; + // Only for the logs: the job logs the start of a wait once, not on each poll. + let mut holding = false; + + loop { + if self.job_token.is_cancelled() { + tracing::info!("Suite {suite_uuid} cancelled; claiming no more tasks"); + return Ok(()); + } + // Check before the claim, not only during a wait: a drain must stop a job that still + // has tasks to claim. This is the difference between a drain and the normal end of + // the suite. + if self.drain_token.is_cancelled() { + tracing::info!( + "Asked to wind down; job {} claims no more tasks", + self.job_id + ); + return Ok(()); + } + + if staged.is_empty() { + let resp = self.fetch_tasks(suite_uuid).await?; + if resp.tasks.is_empty() { + // The job ends when no slot runs a task (`pushed == completed`) and the + // suite gives no more tasks. + if !resp.hold_job_open && progress.completed() == pushed { + tracing::debug!("Suite {suite_uuid} is idle and has no more tasks"); + return Ok(()); + } + if !holding { + // The suite has no ready task, or this job holds all the tasks that it + // can hold. In both cases the wait ends when something changes. + tracing::debug!( + "Nothing to claim from suite {suite_uuid} for now; job {} stays up \ + for more work", + self.job_id + ); + holding = true; + } + // The coordinator notifies this job about a new task in the suite. Thus the + // wake usually ends the wait. The poll is for a dropped socket and for a task + // that a slot of this job submitted during the notification. + tokio::select! { + _ = self.job_token.cancelled() => return Ok(()), + // The checks at the top of the loop also see this. The branch makes + // sure that a drain does not wait for the poll interval first. + _ = self.drain_token.cancelled() => return Ok(()), + _ = wake.changed() => { + tracing::debug!( + "Job {} was notified that suite {suite_uuid} has tasks", + self.job_id + ); + } + // A slot is free. The coordinator possibly gives a task now. + _ = progress.freed.notified() => {} + _ = tokio::time::sleep(HOLD_POLL_INTERVAL) => {} + } + continue; + } + if holding { + tracing::debug!( + "Suite {suite_uuid} has work for job {} again; it resumes", + self.job_id + ); + holding = false; + } + staged.extend(resp.tasks); + } + + // Give the staged tasks one at a time. `send` waits when the channel is full and + // continues when a slot takes a task. Thus the fetch above runs only when the buffer + // is empty. + while let Some(task) = staged.pop_front() { + tokio::select! { + _ = self.job_token.cancelled() => return Ok(()), + _ = self.drain_token.cancelled() => return Ok(()), + sent = task_tx.send(task) => { + if sent.is_err() { + // All the slots stopped. + return Ok(()); + } + pushed += 1; + } + } + } + } + } + + /// One task slot: run the tasks from the buffer in `task-{slot}` until the channel closes or + /// the job drains. + /// + /// A drain finishes only the task in hand. To close the channel is not sufficient, because a + /// receiver still gets the tasks that were in the channel before the close. Thus the slot + /// checks the drain before each task, and `Complete` reclaims the tasks that stay in the + /// channel. + async fn run_slot( + &self, + job: i64, + slot: usize, + task_rx: MAsyncRx, + progress: Arc, + ) { + let dir = self.cache_path.join(format!("task-{slot}")); + let mut executor = self + .executor( + dir.clone(), + self.job_token.clone(), + CancellationToken::new(), + ) + .await; + loop { + let task = tokio::select! { + // Biased, thus a drain that is set wins over a task in the channel. + biased; + _ = self.job_token.cancelled() => break, + _ = self.drain_token.cancelled() => break, + task = task_rx.recv() => match task { + Ok(task) => task, + Err(_) => break, + }, + }; + + tracing::info!("Slot {} running task {}", slot, task.uuid); + tracing::debug!("Slot {} task {} spec: {:?}", slot, task.uuid, task.spec); + if let Err(e) = reset_workspace(&dir).await { + tracing::error!( + "Slot {} error prepare workspace for task {}: {}", + slot, + task.uuid, + e + ); + progress.task_ended(); + continue; + } + let task_uuid = task.uuid; + if let Err(e) = executor + .execute(ExecKind::Task { job: Some(job) }, task) + .await + { + tracing::error!("Slot {} error executing task {}: {}", slot, task_uuid, e); + } + + // After the task, in all cases: the buffer uses this to know if work still runs, and + // if a slot is free. + progress.task_ended(); + } + tracing::debug!("Slot {slot} of job {} released", self.job_id); + } + + /// Run a hook of the suite until it ends. A hook that the suite does not have is not a + /// failure. + async fn run_hook(&self, suite: &TaskSuiteSpec, hook_type: HookType) -> Result<()> { + // A stopped job starts no more hooks. + if self.job_token.is_cancelled() { + return Ok(()); + } + let Some(hook) = self.hook_task(suite, hook_type)? else { + tracing::debug!("No {hook_type} hook configured for suite {}", suite.uuid); + return Ok(()); + }; + self.run_hook_task(hook_type, hook, CancellationToken::new()) + .await + } + + /// Run a hook with the data from [`Self::hook_task`] under the job token. `stop_token` stops + /// the hook and reports its output. Returns an error if the hook did not complete + /// successfully. + async fn run_hook_task( + &self, + hook_type: HookType, + hook: WorkerTaskResp, + stop_token: CancellationToken, + ) -> Result<()> { + tracing::info!("Running the {hook_type} hook of job {}", self.job_id); + let dir = self.cache_path.join(format!("hook-{hook_type}")); + reset_workspace(&dir).await?; + let mut executor = self.executor(dir, self.job_token.clone(), stop_token).await; + match executor.execute(ExecKind::Hook, hook).await? { + // Stopped before the commit: the job stops, and the hook did not fail. + None => Ok(()), + // The same rule as the coordinator: a hook that exited with 0 completed. A stopped + // background hook must also exit with 0. A different message means that the process + // did not run or ran out of time. + Some(result) + if result.exit_status == 0 + && matches!( + result.msg, + None | Some(TaskResultMessage::UploadResultTimeout) + | Some(TaskResultMessage::StoppedByAgent) + ) => + { + Ok(()) + } + Some(result) => Err(error::Error::Custom(format!( + "the {hook_type} hook ended with status {} ({:?})", + result.exit_status, result.msg + ))), + } + } + + /// Start the background hook if the suite has one. The caller does not wait for it: it runs + /// as long as the tasks run, and `stop_token` stops it. + fn spawn_background_hook( + &self, + suite: &TaskSuiteSpec, + stop_token: CancellationToken, + ) -> Option> { + let hook = match self.hook_task(suite, HookType::Background) { + Ok(hook) => hook?, + Err(e) => { + tracing::error!("Background hook failed for suite {}: {e}", suite.uuid); + return None; + } + }; + let runner = self.clone(); + let suite_uuid = suite.uuid; + Some(tokio::spawn(async move { + if let Err(e) = runner + .run_hook_task(HookType::Background, hook, stop_token) + .await + { + tracing::error!("Background hook failed for suite {suite_uuid}: {e}"); + } + })) + } + + /// The data to run a hook: the spec from the suite, and the id and uuid of its hook row. The + /// hook uses the group of the suite for its attachments. `None` if the suite has no such hook. + fn hook_task( + &self, + suite: &TaskSuiteSpec, + hook_type: HookType, + ) -> Result> { + let spec = suite.exec_hooks.as_ref().and_then(|hooks| match hook_type { + HookType::Provision => hooks.provision.clone(), + HookType::Cleanup => hooks.cleanup.clone(), + HookType::Background => hooks.background.clone(), + }); + let Some(spec) = spec else { + return Ok(None); + }; + let row = self + .hooks + .iter() + .find(|hook| hook.hook_type == hook_type) + .ok_or_else(|| { + error::Error::Custom(format!( + "the coordinator sent no row for the {hook_type} hook" + )) + })?; + Ok(Some(WorkerTaskResp { + id: row.id, + uuid: row.uuid, + upstream_task_uuid: None, + group_name: suite.group_name.clone(), + spec, + exec_options: None, + })) + } + + async fn fetch_tasks(&self, suite_uuid: Uuid) -> Result { + let resp = self + .http_client + .get(self.api_url("agents/tasks").as_str()) + .bearer_auth(&self.token) + .query(&FetchTasksReq { suite_uuid }) + .send() + .await + .map_err(error::map_reqwest_err)?; + // The suite stops giving tasks (400), or it is no longer for this agent (404). This is a + // normal end of the job, not a failure: stop the claims and continue to the cleanup. A + // drain and not a cancel of the job, because a cancel also stops the cleanup hook. A + // forced stop comes as its own notification. + if resp.status() == StatusCode::BAD_REQUEST || resp.status() == StatusCode::NOT_FOUND { + tracing::info!("Suite {suite_uuid} stopped handing out tasks"); + self.drain_token.cancel(); + return Ok(FetchTasksResp { + tasks: Vec::new(), + hold_job_open: false, + }); + } + parse_json(resp, "fetch tasks").await + } + + /// Report a phase of the job. Retry until the coordinator gives an answer that ends the + /// report. + /// + /// The agent frees itself from the job also when the report fails. If a report is lost, the + /// coordinator keeps the agent on the job. Thus, different from the task reports, this + /// function ignores `job_token`, and it retries a 5xx and a failed send, which is the answer + /// of a coordinator that restarts. A 4xx ends it. + /// + /// Returns `None` on 404: the job is closed (a user stopped it with force, or a first + /// `Complete` arrived but its answer was lost). Then the job stops, and the cleanup hook does + /// not run. + async fn report_job(&self, job: i64, op: JobReportOp) -> Result> { + let url = self.api_url("agents/job"); + let req = JobReportReq { job, op }; + loop { + let resp = self + .http_client + .post(url.as_str()) + .bearer_auth(&self.token) + .json(&req) + .send() + .await; + let failure = match resp { + Ok(resp) if resp.status() == StatusCode::NOT_FOUND => { + tracing::info!("Job {} is closed; stopping it", self.job_id); + self.job_token.cancel(); + return Ok(None); + } + Ok(resp) if resp.status().is_server_error() => { + format!("coordinator answered {}", resp.status()) + } + Ok(resp) => return parse_json(resp, "job report").await.map(Some), + Err(e) if e.is_builder() => return Err(error::map_reqwest_err(e).into()), + Err(e) => e.to_string(), + }; + tracing::warn!( + "Job report failed ({failure}); retrying in {:?}", + self.connect_retry_interval + ); + tokio::select! { + _ = self.shutdown_token.cancelled() => { + return Err(error::Error::Custom(format!( + "job report abandoned on shutdown: {failure}" + ))); + } + _ = tokio::time::sleep(self.connect_retry_interval) => {} + } + } + } +} + +/// Decode a successful JSON response. A status that is not 2xx becomes an error with the message +/// of the coordinator. +async fn parse_json( + resp: reqwest::Response, + what: &str, +) -> Result { + if !resp.status().is_success() { + return Err(error::Error::Custom(format!( + "{what} failed: {}", + error::get_error_from_resp(resp).await + ))); + } + resp.json::() + .await + .map_err(|e| error::Error::Custom(format!("Unreadable {what} response: {e}"))) +} diff --git a/netmito/src/config/agent.rs b/netmito/src/config/agent.rs new file mode 100644 index 00000000..45dcbc23 --- /dev/null +++ b/netmito/src/config/agent.rs @@ -0,0 +1,427 @@ +use std::collections::HashSet; +use std::io::Write; +use std::ops::Not; +use std::time::Duration; + +use clap::Args; +use figment::{ + providers::{Env, Format, Serialized, Toml}, + value::magic::RelativePathBuf, + Figment, +}; +use serde::{Deserialize, Serialize}; +use tracing_appender::rolling::{RollingFileAppender, Rotation}; +use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt, util::SubscriberInitExt, Layer}; +use url::Url; + +use crate::error::Error; + +use super::{coordinator::DEFAULT_COORDINATOR_ADDR, TracingGuard}; + +/// A writer that adds the agent UUID at the start of each log line. +struct AgentIdWriter { + inner: W, + prefix: Vec, + at_line_start: bool, + buffer: Vec, +} + +impl Drop for AgentIdWriter { + fn drop(&mut self) { + let _ = self.flush_buffer(); + } +} + +impl AgentIdWriter { + fn new(inner: W, agent_id: String) -> Self { + // Make the prefix once. Then each line does not format it again. + let prefix = format!("[agent:{}] ", agent_id).into_bytes(); + Self { + inner, + prefix, + at_line_start: true, + buffer: Vec::with_capacity(8192), + } + } + + fn flush_buffer(&mut self) -> std::io::Result<()> { + if !self.buffer.is_empty() { + self.inner.write_all(&self.buffer)?; + self.buffer.clear(); + } + Ok(()) + } +} + +impl Write for AgentIdWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let mut remaining = buf; + let total_len = buf.len(); + + while !remaining.is_empty() { + // Find the next newline. + if let Some(newline_pos) = remaining.iter().position(|&b| b == b'\n') { + // Write the prefix at the start of a line. + if self.at_line_start && newline_pos > 0 { + self.buffer.extend_from_slice(&self.prefix); + self.at_line_start = false; + } + + // Write the line and its newline. + self.buffer.extend_from_slice(&remaining[..=newline_pos]); + self.at_line_start = true; + + // Flush the buffer when it is larger than 4 KB. + if self.buffer.len() > 4096 { + self.flush_buffer()?; + } + + remaining = &remaining[newline_pos + 1..]; + } else { + // The remaining data has no newline. + if self.at_line_start && !remaining.is_empty() { + self.buffer.extend_from_slice(&self.prefix); + self.at_line_start = false; + } + self.buffer.extend_from_slice(remaining); + break; + } + } + + Ok(total_len) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.flush_buffer()?; + self.inner.flush() + } +} + +/// A `MakeWriter` that makes `AgentIdWriter` instances. +struct AgentIdMakeWriter { + inner: M, + agent_id: String, +} + +impl<'a, M> MakeWriter<'a> for AgentIdMakeWriter +where + M: MakeWriter<'a>, +{ + type Writer = AgentIdWriter; + + fn make_writer(&'a self) -> Self::Writer { + AgentIdWriter::new(self.inner.make_writer(), self.agent_id.clone()) + } +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct AgentConfig { + pub(crate) coordinator_addr: Url, + pub(crate) credential_path: Option, + pub(crate) user: Option, + pub(crate) password: Option, + /// The group that gets the Admin role on the agent. Only an Admin of this group can shut the + /// agent down, and the user that registers the agent must be one. The default is the personal + /// group of the user. + #[serde(default)] + pub(crate) admin_group: Option, + /// The groups that get the Write role on the agent. + pub(crate) groups: HashSet, + /// The tags that a suite must match. The agent can run a suite if it has all the tags of the + /// suite. + pub(crate) tags: HashSet, + /// Labels for queries. Suite matching does not use them. + pub(crate) labels: HashSet, + #[serde(with = "humantime_serde")] + pub(crate) heartbeat_interval: Duration, + /// The time to wait before the agent tries again a coordinator call that could not connect. + #[serde(with = "humantime_serde")] + pub(crate) connect_retry_interval: Duration, + /// The interval at which an idle agent asks the coordinator for a suite with no notification. + /// `None` stops the poll: the agent then asks only after a notification from the heartbeat + /// or the WebSocket. + #[serde(with = "humantime_serde")] + pub(crate) idle_poll_interval: Option, + /// The time to wait before the agent opens a dropped notification WebSocket again. + #[serde(with = "humantime_serde")] + pub(crate) ws_reconnect_interval: Duration, + #[serde(default)] + pub(crate) no_ws: bool, + /// The lifetime of the agent token. `None` means the token never expires. + #[serde(default, with = "humantime_serde")] + pub(crate) lifetime: Option, + /// The machine code. If it is not set, the agent gets one from its cache, from + /// `/etc/machine-id`, or makes a new one. + #[serde(default)] + pub(crate) machine_code: Option, + pub(crate) log_path: Option, + pub(crate) file_log: bool, + #[serde(default)] + pub(crate) shared_log: bool, + #[serde(default)] + pub(crate) skip_redis: bool, +} + +#[derive(Args, Debug, Serialize, Default, Clone)] +#[command(rename_all = "kebab-case")] +pub struct AgentConfigCli { + /// The path of the config file + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub config: Option, + /// The address of the coordinator + #[arg(short, long = "coordinator")] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub coordinator_addr: Option, + /// The path of the user credential file + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub credential_path: Option, + /// The username of the user + #[arg(short, long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub user: Option, + /// The password of the user + #[arg(short, long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub password: Option, + /// The group that gets the Admin role on the agent and can shut it down. The default is the + /// personal group of the user + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub admin_group: Option, + /// The groups that get the Write role on the agent + #[arg(short, long, num_args = 0.., value_delimiter = ',')] + #[serde(skip_serializing_if = "::std::vec::Vec::is_empty")] + pub groups: Vec, + /// The tags that task suites must match + #[arg(short, long, num_args = 0.., value_delimiter = ',')] + #[serde(skip_serializing_if = "::std::vec::Vec::is_empty")] + pub tags: Vec, + /// The labels of this agent + #[arg(short, long, num_args = 0.., value_delimiter = ',')] + #[serde(skip_serializing_if = "::std::vec::Vec::is_empty")] + pub labels: Vec, + /// The interval to send heartbeats (e.g. "30s", "1m") + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub heartbeat_interval: Option, + /// The time between tries of a coordinator call that could not connect (e.g. "30s") + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub connect_retry_interval: Option, + /// The interval at which an idle agent asks for a suite (e.g. "5s"). If not set, the agent + /// waits for a notification + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub idle_poll_interval: Option, + /// The time before the agent opens a dropped notification WebSocket again (e.g. "5s") + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub ws_reconnect_interval: Option, + /// Get notifications only from the heartbeat, and open no notification WebSocket + #[arg(long)] + #[serde(skip_serializing_if = "<&bool>::not")] + pub no_ws: bool, + /// The lifetime of the agent token (e.g. 7d, 1year). If not given, the agent token does not expire + #[arg(long, value_parser = humantime_serde::re::humantime::parse_duration)] + #[serde( + with = "humantime_serde", + skip_serializing_if = "::std::option::Option::is_none" + )] + pub lifetime: Option, + /// The machine code. It replaces the code from /etc/machine-id + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub machine_code: Option, + /// The path of the log file. If not given, the agent uses the default rolling log file + #[arg(long)] + #[serde(skip_serializing_if = "::std::option::Option::is_none")] + pub log_path: Option, + /// Write the logs to a file + #[arg(long)] + #[serde(skip_serializing_if = "<&bool>::not")] + pub file_log: bool, + /// Write the logs of all agents to one file, with a new file each day (3 files at most) + #[arg(long)] + #[serde(skip_serializing_if = "<&bool>::not")] + pub shared_log: bool, + /// Do not connect to Redis + #[arg(long)] + #[serde(skip_serializing_if = "<&bool>::not")] + pub skip_redis: bool, +} + +impl Default for AgentConfig { + fn default() -> Self { + Self { + coordinator_addr: Url::parse(&format!("http://{DEFAULT_COORDINATOR_ADDR}")).unwrap(), + credential_path: None, + user: None, + password: None, + admin_group: None, + groups: HashSet::new(), + tags: HashSet::new(), + labels: HashSet::new(), + heartbeat_interval: Duration::from_secs(60), + connect_retry_interval: Duration::from_secs(30), + idle_poll_interval: None, + ws_reconnect_interval: Duration::from_secs(5), + no_ws: false, + lifetime: None, + machine_code: None, + log_path: None, + file_log: false, + shared_log: false, + skip_redis: false, + } + } +} + +impl AgentConfig { + pub fn new(cli: &AgentConfigCli) -> crate::error::Result { + let global_config = dirs::config_dir().map(|mut p| { + p.push("mitosis"); + p.push("config.toml"); + p + }); + let mut figment = Figment::new().merge(Serialized::from(Self::default(), "agent")); + if let Some(global_config) = global_config { + if global_config.exists() { + figment = figment.merge(Toml::file(global_config).nested()); + } + } + figment = figment + .merge(Toml::file(cli.config.as_deref().unwrap_or("config.toml")).nested()) + .merge(Env::prefixed("MITO_").profile("agent")) + .merge(Serialized::from(cli, "agent")) + .select("agent"); + Ok(figment.extract()?) + } + + pub fn setup_tracing_subscriber(&self, agent_id: U) -> crate::error::Result + where + T: std::fmt::Display, + U: Into, + { + let (file_layer, file_guard) = if self.file_log { + let id = agent_id.into(); + let id_str = id.to_string(); + + // `shared_log` selects the file logger. + let file_logger = if self.shared_log { + // A shared log: use `log_path` if it is given, else "agents.log" + self.log_path + .as_ref() + .and_then(|p| { + let path = p.relative(); + let dir = path.parent(); + let file_name = path.file_name(); + match (dir, file_name) { + (Some(dir), Some(file_name)) => { + // A new file each day, 3 files at most. + RollingFileAppender::builder() + .rotation(Rotation::DAILY) + .filename_prefix(file_name.to_string_lossy().to_string()) + .max_log_files(3) + .build(dir) + .ok() + } + _ => None, + } + }) + .or_else(|| { + // "agents.log" in the cache directory, with a new file each day. + dirs::cache_dir() + .map(|mut p| { + p.push("mitosis"); + p.push("agent"); + p + }) + .and_then(|dir| { + RollingFileAppender::builder() + .rotation(Rotation::DAILY) + .filename_prefix("agents.log") + .max_log_files(3) + .build(dir) + .ok() + }) + }) + .ok_or(Error::ConfigError(Box::new(figment::Error::from( + "log path not valid and cache directory not found", + ))))? + } else { + // A log for this agent only, with no rotation. + self.log_path + .as_ref() + .and_then(|p| { + let path = p.relative(); + let dir = path.parent(); + let file_name = path.file_name(); + match (dir, file_name) { + (Some(dir), Some(file_name)) => { + Some(tracing_appender::rolling::never(dir, file_name)) + } + _ => None, + } + }) + .or_else(|| { + dirs::cache_dir() + .map(|mut p| { + p.push("mitosis"); + p.push("agent"); + p + }) + .map(|dir| { + tracing_appender::rolling::never(dir, format!("{id_str}.log")) + }) + }) + .ok_or(Error::ConfigError(Box::new(figment::Error::from( + "log path not valid and cache directory not found", + ))))? + }; + + let (non_blocking, guard) = tracing_appender::non_blocking(file_logger); + let env_filter = tracing_subscriber::EnvFilter::try_from_env("MITO_FILE_LOG_LEVEL") + .unwrap_or_else(|_| "netmito=info".into()); + + // With `shared_log`, one file holds the logs of many agents, thus each line has its + // agent id. `BoxMakeWriter` gives the two writers one type, thus the two cases use the + // same layers below. + let writer = if self.shared_log { + tracing_subscriber::fmt::writer::BoxMakeWriter::new(AgentIdMakeWriter { + inner: non_blocking, + agent_id: id_str, + }) + } else { + tracing_subscriber::fmt::writer::BoxMakeWriter::new(non_blocking) + }; + let layer = tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true) + .with_writer(writer) + .with_filter(env_filter); + (Some(layer), Some(guard)) + } else { + (None, None) + }; + + tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true) + .with_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "netmito=info".into()), + ), + ) + .with(file_layer) + // The same tracing-subscriber workaround as in `setup_tracing_subscriber` of the + // coordinator. See there for the reason. Remove all of them when tokio-rs/tracing#2519 + // is fixed. + .with(tracing_subscriber::layer::Identity::new()) + .init(); + Ok(TracingGuard { file_guard }) + } +} diff --git a/netmito/src/config/mod.rs b/netmito/src/config/mod.rs index ace92071..873ef5c0 100644 --- a/netmito/src/config/mod.rs +++ b/netmito/src/config/mod.rs @@ -1,7 +1,9 @@ +pub mod agent; pub mod client; pub mod coordinator; pub mod manager; pub mod worker; +pub use agent::{AgentConfig, AgentConfigCli}; pub use client::{ClientConfig, ClientConfigCli}; pub use coordinator::{CoordinatorConfig, CoordinatorConfigCli, InfraPool}; pub(crate) use coordinator::{ diff --git a/netmito/src/entity/state.rs b/netmito/src/entity/state.rs index 6c28cb0d..2c84d390 100644 --- a/netmito/src/entity/state.rs +++ b/netmito/src/entity/state.rs @@ -116,28 +116,28 @@ impl TaskState { TaskState::Cancelled => match result { Some(result_spec) => match result_spec.msg { Some(msg) => matrix_match!( - (target_state, msg); TaskResultMessage::FetchResourceTimeout, TaskResultMessage::ExecTimeout, TaskResultMessage::UploadResultTimeout, TaskResultMessage::ResourceNotFound, TaskResultMessage::ResourceForbidden, TaskResultMessage::WatchTimeout, TaskResultMessage::UserCancellation, TaskResultMessage::SubmitNewTaskFailed => - TaskExecState::WorkerExited => false, false, false, false, false, false, false, false; - TaskExecState::FetchResource => false, false, false, false, false, false, false, true; - TaskExecState::FetchResourceFinished => false, false, false, false, false, false, false, true; - TaskExecState::FetchResourceError => false, false, false, false, false, false, false, false; - TaskExecState::FetchResourceTimeout => true, false, false, false, false, false, false, false; - TaskExecState::FetchResourceNotFound => false, false, false, true, false, false, false, false; - TaskExecState::FetchResourceForbidden => false, false, false, false, true, false, false, false; - TaskExecState::Watch => false, false, false, false, false, false, false, true; - TaskExecState::WatchFinished => false, false, false, false, false, false, false, true; - TaskExecState::WatchTimeout => false, false, false, false, false, true, false, false; - TaskExecState::ExecPending => false, false, false, false, false, false, false, true; - TaskExecState::ExecSpawned => false, false, false, false, false, false, false, true; - TaskExecState::ExecFinished => false, false, false, false, false, false, false, true; - TaskExecState::ExecTimeout => false, true, false, false, false, false, false, false; - TaskExecState::UploadResult => true, true, true, true, true, true, false, true; - TaskExecState::UploadFinishedResult => false, false, false, false, false, false, false, false; - TaskExecState::UploadCancelledResult => true, true, true, true, true, true, false, false; - TaskExecState::UploadResultFinished => true, true, true, true, true, true, false, true; - TaskExecState::UploadResultTimeout => false, false, true, false, false, false, false, false; - TaskExecState::TaskCommitted => true, true, true, true, true, true, true, true; - TaskExecState::Unknown => false, false, false, false, false, false, false, false; + (target_state, msg); TaskResultMessage::FetchResourceTimeout, TaskResultMessage::ExecTimeout, TaskResultMessage::UploadResultTimeout, TaskResultMessage::ResourceNotFound, TaskResultMessage::ResourceForbidden, TaskResultMessage::WatchTimeout, TaskResultMessage::UserCancellation, TaskResultMessage::SubmitNewTaskFailed, TaskResultMessage::StoppedByAgent => + TaskExecState::WorkerExited => false, false, false, false, false, false, false, false, false; + TaskExecState::FetchResource => false, false, false, false, false, false, false, true, false; + TaskExecState::FetchResourceFinished => false, false, false, false, false, false, false, true, false; + TaskExecState::FetchResourceError => false, false, false, false, false, false, false, false, false; + TaskExecState::FetchResourceTimeout => true, false, false, false, false, false, false, false, false; + TaskExecState::FetchResourceNotFound => false, false, false, true, false, false, false, false, false; + TaskExecState::FetchResourceForbidden => false, false, false, false, true, false, false, false, false; + TaskExecState::Watch => false, false, false, false, false, false, false, true, false; + TaskExecState::WatchFinished => false, false, false, false, false, false, false, true, false; + TaskExecState::WatchTimeout => false, false, false, false, false, true, false, false, false; + TaskExecState::ExecPending => false, false, false, false, false, false, false, true, false; + TaskExecState::ExecSpawned => false, false, false, false, false, false, false, true, false; + TaskExecState::ExecFinished => false, false, false, false, false, false, false, true, false; + TaskExecState::ExecTimeout => false, true, false, false, false, false, false, false, false; + TaskExecState::UploadResult => true, true, true, true, true, true, false, true, false; + TaskExecState::UploadFinishedResult => false, false, false, false, false, false, false, false, false; + TaskExecState::UploadCancelledResult => true, true, true, true, true, true, false, false, false; + TaskExecState::UploadResultFinished => true, true, true, true, true, true, false, true, false; + TaskExecState::UploadResultTimeout => false, false, true, false, false, false, false, false, false; + TaskExecState::TaskCommitted => true, true, true, true, true, true, true, true, true; + TaskExecState::Unknown => false, false, false, false, false, false, false, false, false; ), None => false, }, diff --git a/netmito/src/executor/mod.rs b/netmito/src/executor/mod.rs index 5987d3e6..fabedec6 100644 --- a/netmito/src/executor/mod.rs +++ b/netmito/src/executor/mod.rs @@ -1,5 +1,5 @@ -//! Runs one task. The executor gets the task inputs, starts the process, archives -//! and uploads the output, and then commits the result to the coordinator. +//! Runs one task or one hook. The executor gets the inputs, starts the process, +//! archives and uploads the output, and then commits the result to the coordinator. use std::path::PathBuf; use std::process::ExitStatus; @@ -23,6 +23,11 @@ use crate::service::s3::download_file; mod report; mod state; mod upload; +mod workspace; + +pub use workspace::{ + create_shared_dir, ensure_traversable_dir, reset_workspace, warn_unreachable_ancestors, +}; // Default time budgets. Each executor keeps its own copy, so a runner can set // different budgets on each executor that it builds. @@ -33,14 +38,44 @@ pub(crate) const SINGLE_DOWNLOAD_BUDGET: std::time::Duration = std::time::Durati /// The maximum time to archive and upload the output after the process stops. pub(crate) const UPLOAD_BUDGET: std::time::Duration = std::time::Duration::from_secs(600); +/// What the executor runs. The kind sets the fields of the reports, and only a task can submit a +/// downstream task. +#[derive(Debug, Clone, Copy)] +pub enum ExecKind { + /// A task. An agent gives the handle of the job that claimed the task. A worker gives `None`. + Task { job: Option }, + /// A hook of an agent job. + Hook, +} + +impl ExecKind { + fn report_req(self, id: i64, op: ReportTaskOp) -> ReportTaskReq { + let (kind, job) = match self { + ExecKind::Task { job } => (TaskKind::Task, job), + ExecKind::Hook => (TaskKind::Hook, None), + }; + ReportTaskReq { id, kind, job, op } + } +} + pub struct Executor { pub(crate) http_client: Client, pub(crate) task_credential: String, pub(crate) http_url: Url, + /// Stops the execution. The executor kills the process and reports nothing. pub(crate) task_cancel_token: CancellationToken, + /// Stops the process, then reports it with its output: a SIGTERM to the process group, and a + /// SIGKILL if the process does not stop in a grace time. The result has the message + /// `StoppedByAgent`. A worker gives a token that it does not cancel. + pub(crate) task_stop_token: CancellationToken, + /// Stops the worker or the agent. The executor cancels it when the coordinator rejects the + /// credential. A worker gives the same token as `task_cancel_token`. + pub(crate) runner_cancel_token: CancellationToken, pub(crate) coordinator_force_exit: Arc, pub(crate) polling_interval: std::time::Duration, pub(crate) task_cache_path: PathBuf, + /// More environment variables for the process. + pub(crate) extra_envs: Vec<(String, String)>, pub(crate) task_redis_conn: Option, pub(crate) task_redis_pubsub: Option, pub(crate) resource_fetch_budget: std::time::Duration, @@ -49,7 +84,14 @@ pub struct Executor { } impl Executor { - pub async fn execute(&mut self, task: WorkerTaskResp) -> crate::error::Result<()> { + /// Runs `task` and reports it as `kind`. For a hook, `task` holds the data of the hook. + /// + /// Returns the committed result, or `None` if the execution stopped before the commit. + pub async fn execute( + &mut self, + kind: ExecKind, + task: WorkerTaskResp, + ) -> crate::error::Result> { self.announce_task_state_ex(&task.uuid, TaskExecState::FetchResource as i32, 360) .await; let timeout_until = tokio::time::Instant::now() + self.resource_fetch_budget; @@ -87,19 +129,18 @@ impl Executor { ); tokio::select! { biased; - _ = self.task_cancel_token.cancelled() => return Ok(()), + _ = self.task_cancel_token.cancelled() => return Ok(None), + _ = self.task_stop_token.cancelled() => return Ok(None), _ = tokio::time::sleep(self.polling_interval) => {}, _ = tokio::time::sleep_until(timeout_until) => { tracing::debug!("Fetching resource timeout, commit this task as canceled"); - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::FetchResourceTimeout), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::FetchResourceTimeout), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel).await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) + .await?; self .announce_task_state_ex( &task.uuid, @@ -114,7 +155,7 @@ impl Executor { 60, ) .await; - return Ok(()); + return Ok(Some(result)); } } continue; @@ -135,24 +176,20 @@ impl Executor { .json::() .await .map_err(RequestError::from)?; - let local_path = self - .task_cache_path - .join("resource") - .join(resource.local_path); + let resource_root = self.task_cache_path.join("resource"); + let local_path = resource_root.join(resource.local_path); tokio::select! { biased; - res = download_file(&self.http_client, &download_resp, local_path, false) => { + res = download_file(&self.http_client, &download_resp, &local_path, false) => { if let Err(e) = res { tracing::error!("Failed to download resource: {}", e); - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::ResourceForbidden), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::ResourceForbidden), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel).await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) + .await?; self .announce_task_state( &task.uuid, @@ -166,21 +203,21 @@ impl Executor { 60, ) .await; - return Ok(()); + return Ok(Some(result)); } + workspace::share_fetched_resource(&resource_root, &local_path).await; } - _ = self.task_cancel_token.cancelled() => return Ok(()), + _ = self.task_cancel_token.cancelled() => return Ok(None), + _ = self.task_stop_token.cancelled() => return Ok(None), _ = tokio::time::sleep(self.single_download_budget) => { tracing::debug!("Fetching resource timeout, commit this task as canceled"); - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::FetchResourceTimeout), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::FetchResourceTimeout), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel).await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) + .await?; self .announce_task_state_ex( &task.uuid, @@ -191,19 +228,17 @@ impl Executor { self .announce_task_state_ex(&task.uuid, TaskExecState::TaskCommitted as i32, 60) .await; - return Ok(()); + return Ok(Some(result)); } _ = tokio::time::sleep_until(timeout_until) => { tracing::debug!("Fetching resource timeout, commit this task as canceled"); - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::FetchResourceTimeout), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::FetchResourceTimeout), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel).await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) + .await?; self .announce_task_state_ex( &task.uuid, @@ -214,20 +249,19 @@ impl Executor { self .announce_task_state_ex(&task.uuid, TaskExecState::TaskCommitted as i32, 60) .await; - return Ok(()); + return Ok(Some(result)); } } } else if resp.status() == StatusCode::NOT_FOUND { tracing::debug!("Resource not found, commit this task as canceled"); - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::ResourceNotFound), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::ResourceNotFound), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel) + .await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) + .await?; self.announce_task_state_ex( &task.uuid, TaskExecState::FetchResourceNotFound as i32, @@ -236,20 +270,19 @@ impl Executor { .await; self.announce_task_state_ex(&task.uuid, TaskExecState::TaskCommitted as i32, 60) .await; - return Ok(()); + return Ok(Some(result)); } else if resp.status() == StatusCode::FORBIDDEN { tracing::debug!( "Resource is forbidden to be fetched, commit this task as canceled" ); - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::ResourceForbidden), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::ResourceForbidden), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel) + .await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) + .await?; self.announce_task_state_ex( &task.uuid, TaskExecState::FetchResourceForbidden as i32, @@ -258,7 +291,7 @@ impl Executor { .await; self.announce_task_state_ex(&task.uuid, TaskExecState::TaskCommitted as i32, 60) .await; - return Ok(()); + return Ok(Some(result)); } else { let resp: ErrorMsg = resp .json() @@ -281,45 +314,42 @@ impl Executor { task.exec_options.as_ref().and_then(|o| o.watch) { // Wait until the watched task reaches the specified state. Then start this task. - if self.task_redis_conn.is_some() && self.task_redis_pubsub.is_some() { - self.announce_task_state(&task.uuid, TaskExecState::Watch as i32) - .await; - let tmp_cancel_token = self.task_cancel_token.clone(); - tokio::select! { - biased; - _ = tmp_cancel_token.cancelled() => { - tracing::info!("Task watching interrupted by shutdown signal"); - self.unsubscribe_task_exec_state(&watched_task_uuid).await; - self - .announce_task_state_ex(&task.uuid, TaskExecState::WorkerExited as i32, 60) - .await; - return Ok(()); - }, - _ = self.watch_task(&watched_task_uuid, watched_task_state) => {}, - _ = tokio::time::sleep_until(timeout_until) => { - tracing::debug!("Watching timeout, commit this task as canceled"); - self.unsubscribe_task_exec_state(&watched_task_uuid).await; - self.report_task(task.id, ReportTaskOp::Cancel).await?; - self.report_task( - task.id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: 0, - msg: Some(TaskResultMessage::WatchTimeout), - }), - ) + // Without Redis, the executor asks the coordinator for the state at intervals. + self.announce_task_state(&task.uuid, TaskExecState::Watch as i32) + .await; + let tmp_cancel_token = self.task_cancel_token.clone(); + tokio::select! { + biased; + _ = tmp_cancel_token.cancelled() => { + tracing::info!("Task watching interrupted by shutdown signal"); + self.unsubscribe_task_exec_state(&watched_task_uuid).await; + self + .announce_task_state_ex(&task.uuid, TaskExecState::WorkerExited as i32, 60) + .await; + return Ok(None); + }, + _ = self.watch_task(&watched_task_uuid, watched_task_state) => {}, + _ = tokio::time::sleep_until(timeout_until) => { + tracing::debug!("Watching timeout, commit this task as canceled"); + self.unsubscribe_task_exec_state(&watched_task_uuid).await; + let result = TaskResultSpec { + exit_status: 0, + msg: Some(TaskResultMessage::WatchTimeout), + }; + self.report_task(kind, task.id, ReportTaskOp::Cancel).await?; + self.report_task(kind, task.id, ReportTaskOp::Commit(result.clone())) .await?; - self - .announce_task_state_ex( - &task.uuid, - TaskExecState::WatchTimeout as i32, - 60, - ) - .await; - self - .announce_task_state_ex(&task.uuid, TaskExecState::TaskCommitted as i32, 60) - .await; - return Ok(()); - } + self + .announce_task_state_ex( + &task.uuid, + TaskExecState::WatchTimeout as i32, + 60, + ) + .await; + self + .announce_task_state_ex(&task.uuid, TaskExecState::TaskCommitted as i32, 60) + .await; + return Ok(Some(result)); } } } @@ -348,12 +378,20 @@ impl Executor { command .args(task.spec.args) .envs(task.spec.envs) + .envs(self.extra_envs.iter().cloned()) .env("MITO_RESULT_DIR", self.task_cache_path.join("result")) .env("MITO_EXEC_DIR", self.task_cache_path.join("exec")) .env("MITO_RESOURCE_DIR", self.task_cache_path.join("resource")) .env("MITO_TASK_UUID", task.uuid.to_string()) - .env("MITO_NEW_TASK", &new_task_path) .stdin(std::process::Stdio::null()); + // The task leads its own process group. Thus a kill of the full group from the task (for + // example, a task that removes its namespaces when it stops) does not reach the worker or + // the agent. The timeout SIGTERM also reaches all processes of the task. + #[cfg(unix)] + command.process_group(0); + if let ExecKind::Task { .. } = kind { + command.env("MITO_NEW_TASK", &new_task_path); + } if let Some(uuid) = task.upstream_task_uuid { command.env("MITO_UPSTREAM_TASK_UUID", uuid.to_string()); } @@ -367,12 +405,19 @@ impl Executor { .stderr(std::process::Stdio::null()); } + // A stop before the process starts ends the execution with no report, as a cancel does. + if self.task_stop_token.is_cancelled() { + return Ok(None); + } let mut child = command.spawn().inspect_err(|e| { tracing::error!("Failed to spawn task: {}", e); })?; + let pid = child.id(); self.announce_task_state(&task.uuid, TaskExecState::ExecSpawned as i32) .await; - let process_output = async { + // Boxed, thus the executor can drop it to use `child` again. After a SIGTERM, the executor + // continues to await this future. Thus the output that it read before the signal stays. + let mut process_output = Box::pin(async { if task.spec.terminal_output { let process_output = async { let mut stdout_buf = Vec::new(); @@ -397,75 +442,119 @@ impl Executor { .await .map(|exit_status| ProcessOutput::WithoutLog { exit_status }) } - }; + }); - let output = tokio::select! { + let end = tokio::select! { biased; _ = self.task_cancel_token.cancelled() => { tracing::info!("Task execution interrupted by shutdown signal"); + drop(process_output); + // The task has its own process group, thus a Ctrl-C at the terminal does not reach + // it. Kill the group, or the processes of the task continue after the runner stops. + #[cfg(unix)] + if let Some(id) = child.id() { + let _ = signal::kill(Pid::from_raw(-(id as i32)), Signal::SIGKILL); + } child.kill().await.inspect_err(|e| { tracing::error!("Failed to kill task: {}", e); })?; self .announce_task_state_ex(&task.uuid, TaskExecState::WorkerExited as i32, 60) .await; - return Ok(()); + return Ok(None); }, - output = process_output => { - self - .announce_task_state_ex(&task.uuid, TaskExecState::ExecFinished as i32, 660) - .await; - output.map(TaskResult::Finish) - }, - _ = tokio::time::sleep_until(timeout_until) => { - tracing::debug!("Task execution timeout"); - self - .announce_task_state_ex(&task.uuid, TaskExecState::ExecTimeout as i32, 60) + output = &mut process_output => ProcessEnd::Exited(output), + _ = tokio::time::sleep_until(timeout_until) => ProcessEnd::Timeout, + _ = self.task_stop_token.cancelled() => ProcessEnd::Stopped, + }; + let output = match end { + ProcessEnd::Exited(output) => { + self.announce_task_state_ex(&task.uuid, TaskExecState::ExecFinished as i32, 660) .await; - if let Some(id) = child.id() { + TaskResult::Finish(output?) + } + ProcessEnd::Timeout | ProcessEnd::Stopped => { + if let ProcessEnd::Timeout = end { + tracing::debug!("Task execution timeout"); + self.announce_task_state_ex(&task.uuid, TaskExecState::ExecTimeout as i32, 60) + .await; + } else { + tracing::info!("Task execution stopped by the agent"); + } + if let Some(id) = pid { // TODO: Send the signal through a pidfd when `linux_pidfd` is stable. // A raw PID can refer to a different process if the system uses the PID again. // Tracking issue for std lib: [rust-lang/rust #82971](https://github.com/rust-lang/rust/issues/82971) // Tracking issue for tokio: [tokio-rs/tokio #6281](https://github.com/tokio-rs/tokio/issues/6281) - let _ = signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).inspect_err(|e| { - tracing::error!("Failed to send SIGTERM to task: {}", e); - }); + // A negative PID sends the signal to the process group. The task leads its own + // group, thus the signal reaches all processes of the task, not only `env`. + let _ = signal::kill(Pid::from_raw(-(id as i32)), Signal::SIGTERM).inspect_err( + |e| { + tracing::error!( + "Failed to send SIGTERM to the task process group: {}", + e + ); + }, + ); } - tokio::select! { + let stopped = tokio::select! { biased; - _ = child.wait() => {}, + output = &mut process_output => Some(output), _ = self.task_cancel_token.cancelled() => { + drop(process_output); child.kill().await.inspect_err(|e| { tracing::error!("Failed to kill task: {}", e); })?; - return Ok(()); - }, - _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => { - child.kill().await.inspect_err(|e| { - tracing::error!("Failed to kill task: {}", e); - })?; + return Ok(None); }, - } - if task.spec.terminal_output { - let output = child.wait_with_output().await?; - Ok(TaskResult::Timeout(ProcessOutput::WithLog { - stdout: output.stdout, - stderr: output.stderr, - exit_status: output.status, - })) + _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => None, + }; + let output = match stopped { + Some(output) => output?, + None => { + // The process did not stop in the grace time. Kill its group. A process + // that left the group can keep the pipes open, thus wait only a short time + // for the remaining output. + if let Some(id) = pid { + let _ = signal::kill(Pid::from_raw(-(id as i32)), Signal::SIGKILL); + } + let killed = tokio::select! { + output = &mut process_output => Some(output), + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => None, + }; + match killed { + Some(output) => output?, + None => { + drop(process_output); + child.kill().await.inspect_err(|e| { + tracing::error!("Failed to kill task: {}", e); + })?; + let exit_status = child.wait().await?; + if task.spec.terminal_output { + ProcessOutput::WithLog { + stdout: Vec::new(), + stderr: Vec::new(), + exit_status, + } + } else { + ProcessOutput::WithoutLog { exit_status } + } + } + } + } + }; + if let ProcessEnd::Timeout = end { + TaskResult::Timeout(output) } else { - let exit_status = child.wait().await?; - Ok(TaskResult::Timeout(ProcessOutput::WithoutLog { - exit_status, - })) + TaskResult::Stopped(output) } - }, - }?; + } + }; tracing::debug!("Task execution finished"); self.announce_task_state_ex(&task.uuid, TaskExecState::UploadResult as i32, 660) .await; - self.process_task_result(task.id, task.uuid, output).await?; - Ok(()) + self.process_task_result(kind, task.id, task.uuid, output) + .await } } @@ -489,9 +578,18 @@ impl ProcessOutput { } } +/// How the process ended, before the executor collects its output. +enum ProcessEnd { + Exited(std::io::Result), + Timeout, + Stopped, +} + enum TaskResult { Finish(ProcessOutput), Timeout(ProcessOutput), + /// The agent stopped the process with `task_stop_token`. + Stopped(ProcessOutput), } impl TaskResult { @@ -499,6 +597,7 @@ impl TaskResult { match self { TaskResult::Finish(output) => output, TaskResult::Timeout(output) => output, + TaskResult::Stopped(output) => output, } } } diff --git a/netmito/src/executor/report.rs b/netmito/src/executor/report.rs index 8db27432..0a171ac5 100644 --- a/netmito/src/executor/report.rs +++ b/netmito/src/executor/report.rs @@ -5,20 +5,16 @@ use reqwest::StatusCode; use crate::error::{Error, ErrorMsg, RequestError}; use crate::schema::*; -use super::Executor; +use super::{ExecKind, Executor}; impl Executor { pub(super) async fn report_task( &mut self, + kind: ExecKind, id: i64, op: ReportTaskOp, ) -> crate::error::Result<()> { - let req = ReportTaskReq { - id, - kind: TaskKind::Task, - job: None, - op, - }; + let req = kind.report_req(id, op); self.http_url.set_path("tasks/report"); loop { let resp = self @@ -36,7 +32,7 @@ impl Executor { tracing::info!("Report task failed with coordinator force exit"); self.coordinator_force_exit .store(true, std::sync::atomic::Ordering::Release); - self.task_cancel_token.cancel(); + self.runner_cancel_token.cancel(); return Ok(()); } else if resp.status() == StatusCode::NOT_FOUND { tracing::debug!("Task not found, ignore and go on for next cycle"); @@ -76,7 +72,15 @@ impl Executor { /// Submits the downstream task that the process wrote to `MITO_NEW_TASK`. /// Returns false only if the file is present and the submission fails. - pub(super) async fn submit_new_task_if_present(&mut self, task_id: i64) -> bool { + pub(super) async fn submit_new_task_if_present( + &mut self, + kind: ExecKind, + task_id: i64, + ) -> bool { + // A hook has no `MITO_NEW_TASK` and cannot submit a task. + if let ExecKind::Hook = kind { + return true; + } let new_task_path = self.task_cache_path.join("new_task.json"); if !new_task_path.exists() { return true; @@ -112,7 +116,7 @@ impl Executor { submit_req.group_name ); if let Err(e) = self - .report_task(task_id, ReportTaskOp::Submit(Box::new(submit_req))) + .report_task(kind, task_id, ReportTaskOp::Submit(Box::new(submit_req))) .await { tracing::warn!("Failed to submit new task: {}", e); diff --git a/netmito/src/executor/upload.rs b/netmito/src/executor/upload.rs index b9a41b18..1c266d12 100644 --- a/netmito/src/executor/upload.rs +++ b/netmito/src/executor/upload.rs @@ -16,19 +16,25 @@ use crate::entity::state::TaskExecState; use crate::error::{Error, ErrorMsg, RequestError}; use crate::schema::*; -use super::{Executor, ProcessOutput, TaskResult}; +use super::workspace::reset_workspace; +use super::{ExecKind, Executor, ProcessOutput, TaskResult}; impl Executor { + /// Returns the committed result, or `None` if the executor stopped before the commit. pub(super) async fn process_task_result( &mut self, + kind: ExecKind, id: i64, uuid: Uuid, output: TaskResult, - ) -> crate::error::Result<()> { + ) -> crate::error::Result> { + // A stopped process reports `Finish`: its end is the normal end that the agent asked for. let (is_finished, exit_status) = match &output { TaskResult::Finish(output) => (true, output.get_exit_status()), TaskResult::Timeout(output) => (false, output.get_exit_status()), + TaskResult::Stopped(output) => (true, output.get_exit_status()), }; + let stopped = matches!(output, TaskResult::Stopped(_)); let op = if is_finished { self.announce_task_state_ex(&uuid, TaskExecState::UploadFinishedResult as i32, 660) .await; @@ -38,13 +44,16 @@ impl Executor { .await; ReportTaskOp::Cancel }; - self.report_task(id, op).await?; + self.report_task(kind, id, op).await?; // A spawned task archives each output, and this task uploads each archive when it is ready. // The channel sends the content type and size of each archive. let (tx, rx) = crossfire::spsc::unbounded_async::<(ArtifactContentType, u64)>(); let timeout_cancel_token = CancellationToken::new(); let archive_timeout_cancel_token = timeout_cancel_token.clone(); - let archive_cancel_token = self.task_cancel_token.clone(); + // Use a child token. Thus a stop of the archive or of the upload stops only the archive, and + // the job of an agent continues. + let archive_cancel_token = self.task_cancel_token.child_token(); + let stop_archive_token = archive_cancel_token.clone(); let archive_cache_path = self.task_cache_path.clone(); let archive_hd = tokio::spawn(async move { let result_dir = archive_cache_path.join("result"); @@ -226,15 +235,13 @@ impl Executor { // that no upload uses. let rx = rx; while let Ok((content_type, content_length)) = rx.recv().await { - let req = ReportTaskReq { + let req = kind.report_req( id, - kind: TaskKind::Task, - job: None, - op: ReportTaskOp::Upload { + ReportTaskOp::Upload { content_type, content_length, }, - }; + ); self.http_url.set_path("tasks/report"); let resp = loop { let resp = self @@ -258,7 +265,7 @@ impl Executor { ); self.coordinator_force_exit .store(true, std::sync::atomic::Ordering::Release); - self.task_cancel_token.cancel(); + self.runner_cancel_token.cancel(); return Ok(()); } else if resp.status() == StatusCode::NOT_FOUND { tracing::debug!("Task not found, ignore and go on for next cycle"); @@ -278,7 +285,7 @@ impl Executor { .json() .await .unwrap_or_else(|e| ErrorMsg { msg: e.to_string() }); - self.task_cancel_token.cancel(); + stop_archive_token.cancel(); return Err(Error::Custom(format!( "Request upload url failed with error: {}", resp.msg @@ -299,7 +306,7 @@ impl Executor { } continue; } else { - self.task_cancel_token.cancel(); + stop_archive_token.cancel(); return Err(RequestError::from(e).into()); } } @@ -368,19 +375,17 @@ impl Executor { crate::error::Result::Ok(()) }; let timeout_until = tokio::time::Instant::now() + self.upload_budget; - tokio::select! { + let result = tokio::select! { biased; _ = tokio::time::sleep_until(timeout_until) => { tracing::warn!("Upload result timeout"); timeout_cancel_token.cancel(); - self.report_task( - id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: exit_status.into_raw(), - msg: Some(TaskResultMessage::UploadResultTimeout), - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: exit_status.into_raw(), + msg: Some(TaskResultMessage::UploadResultTimeout), + }; + self.report_task(kind, id, ReportTaskOp::Commit(result.clone())) + .await?; self .announce_task_state_ex(&uuid, TaskExecState::UploadResultTimeout as i32, 60) .await; @@ -388,6 +393,7 @@ impl Executor { .announce_task_state_ex(&uuid, TaskExecState::TaskCommitted as i32, 60) .await; archive_hd.await??; + result } res = upload_artifact_fut => { res?; @@ -397,14 +403,16 @@ impl Executor { self .announce_task_state_ex(&uuid, TaskExecState::WorkerExited as i32, 60) .await; - return Ok(()); + return Ok(None); } self .announce_task_state_ex(&uuid, TaskExecState::UploadResultFinished as i32, 60) .await; - let new_task_scceed = self.submit_new_task_if_present(id).await; - let msg = if is_finished { + let new_task_scceed = self.submit_new_task_if_present(kind, id).await; + let msg = if stopped { + Some(TaskResultMessage::StoppedByAgent) + } else if is_finished { if new_task_scceed { None } else { @@ -413,26 +421,21 @@ impl Executor { } else { Some(TaskResultMessage::ExecTimeout) }; - self.report_task( - id, - ReportTaskOp::Commit(TaskResultSpec { - exit_status: exit_status.into_raw(), - msg - }), - ) - .await?; + let result = TaskResultSpec { + exit_status: exit_status.into_raw(), + msg + }; + self.report_task(kind, id, ReportTaskOp::Commit(result.clone())) + .await?; self .announce_task_state_ex(&uuid, TaskExecState::TaskCommitted as i32, 60) .await; - + result } - } + }; // Reset the cache directory for the next task. - tokio::fs::remove_dir_all(&self.task_cache_path).await?; - tokio::fs::create_dir_all(&self.task_cache_path).await?; - tokio::fs::create_dir_all(&self.task_cache_path.join("result")).await?; - tokio::fs::create_dir_all(&self.task_cache_path.join("exec")).await?; - Ok(()) + reset_workspace(&self.task_cache_path).await?; + Ok(Some(result)) } } diff --git a/netmito/src/executor/workspace.rs b/netmito/src/executor/workspace.rs new file mode 100644 index 00000000..cd009456 --- /dev/null +++ b/netmito/src/executor/workspace.rs @@ -0,0 +1,140 @@ +//! Creates the working directories of the executor. A task can change to a different user while +//! it runs, thus the directories that the task writes to are open to all users. + +#[cfg(unix)] +use std::fs::Permissions; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +/// The mode of a directory that the task writes to: all users can read, write and enter it. The +/// sticky bit stops a user from removing the files of a different user. +/// +/// The worker or the agent creates the directories, but the task can change to a different user +/// (`sudo`, a runtime that drops privileges, a nested container). With the usual umask, that user +/// cannot write its results. Thus the executor sets the mode and does not use the umask. +#[cfg(unix)] +const SHARED_DIR_MODE: u32 = 0o1777; + +/// The mode of a directory above the working directory. Only the owner can write to it, but all +/// users can enter it. Thus a task that runs as a different user can get to the directories below. +#[cfg(unix)] +const TRAVERSABLE_DIR_MODE: u32 = 0o755; + +/// The mode of a downloaded input. It is also executable, because an input is frequently the +/// program that the task runs. The sticky bit has no effect on a file, thus it is not set. +#[cfg(unix)] +const SHARED_FILE_MODE: u32 = 0o777; + +/// Create `path` and its missing parents, then let all users read and write `path`. +/// +/// The parents keep the umask. To open a full tree, call this function once for each level, from +/// the top down. +pub async fn create_shared_dir(path: &Path) -> crate::error::Result<()> { + tokio::fs::create_dir_all(path).await?; + set_mode(path, SHARED_DIR_MODE).await; + Ok(()) +} + +/// Create `path` and its missing parents, then let all users enter `path`. Use it for the +/// directories above a working directory: one `0700` directory in the chain stops a task that +/// changed user. +pub async fn ensure_traversable_dir(path: &Path) -> crate::error::Result<()> { + tokio::fs::create_dir_all(path).await?; + set_mode(path, TRAVERSABLE_DIR_MODE).await; + Ok(()) +} + +/// Set `mode` on a path that exists. A failure is only logged: the usual cause is a directory that +/// this process does not own, and only a task that changes user is affected. +#[cfg(unix)] +async fn set_mode(path: &Path, mode: u32) { + if let Err(e) = tokio::fs::set_permissions(path, Permissions::from_mode(mode)).await { + tracing::warn!( + "Failed to set mode {mode:o} on {}: {e}. A task that changes user possibly cannot use it", + path.display() + ); + } +} + +#[cfg(not(unix))] +async fn set_mode(_path: &Path, _mode: u32) {} + +/// Log a warning for each parent of `path` that a task that runs as a different user cannot +/// enter. The executor does not change these directories: the operator decides (the usual case is +/// a `$HOME` with mode `0700`). +#[cfg(unix)] +pub async fn warn_unreachable_ancestors(path: &Path) { + // Collect the paths first. A borrow of `path` across the awaits below makes the future not + // `Send`. + let ancestors: Vec = path.ancestors().skip(1).map(PathBuf::from).collect(); + for ancestor in ancestors { + let Ok(meta) = tokio::fs::metadata(&ancestor).await else { + continue; + }; + let mode = meta.permissions().mode(); + if mode & 0o001 == 0 { + tracing::warn!( + "{} has mode {:o}: a task that changes user cannot get to the working directory \ + below it. Give it at least o+x if such tasks can run", + ancestor.display(), + mode & 0o7777 + ); + } + } +} + +#[cfg(not(unix))] +pub async fn warn_unreachable_ancestors(_path: &Path) {} + +/// Let all users read, write and execute a downloaded input. `download_file` uses the umask, and +/// the client CLI also uses it, where the downloads of a user must stay private. Thus the executor +/// changes the mode here. +/// +/// A `local_path` can name a file in a subdirectory (`bin/tool`). The directories that +/// `download_file` created for it also get the shared mode. If the path is not a plain relative +/// path (`..` or an absolute path), only the file changes, because the change must not go above +/// `resource/`. +#[cfg(unix)] +pub(super) async fn share_fetched_resource(resource_root: &Path, file: &Path) { + set_mode(file, SHARED_FILE_MODE).await; + + let Ok(nested) = file.strip_prefix(resource_root) else { + return; + }; + if !nested + .components() + .all(|c| matches!(c, std::path::Component::Normal(_))) + { + tracing::warn!( + "Do not change the directories of {}: the path goes out of the resource directory", + file.display() + ); + return; + } + // Go up from the file and stop at `resource/`. `reset_workspace` already shares `resource/`. + let dirs: Vec = file + .ancestors() + .skip(1) + .take_while(|dir| *dir != resource_root) + .map(PathBuf::from) + .collect(); + for dir in dirs { + set_mode(&dir, SHARED_DIR_MODE).await; + } +} + +#[cfg(not(unix))] +pub(super) async fn share_fetched_resource(_resource_root: &Path, _file: &Path) {} + +/// Remove the working directory and create it again, with the directories that the process uses. +/// All users can write to them, because the process can change to a different user. +pub async fn reset_workspace(cache_path: &Path) -> crate::error::Result<()> { + let _ = tokio::fs::remove_dir_all(cache_path).await; + // The process writes `new_task.json` to the working directory itself, thus it is also shared. + create_shared_dir(cache_path).await?; + create_shared_dir(&cache_path.join("result")).await?; + create_shared_dir(&cache_path.join("exec")).await?; + create_shared_dir(&cache_path.join("resource")).await?; + Ok(()) +} diff --git a/netmito/src/lib.rs b/netmito/src/lib.rs index eb483dbe..3a6875e7 100644 --- a/netmito/src/lib.rs +++ b/netmito/src/lib.rs @@ -1,3 +1,4 @@ +pub mod agent; pub mod api; pub mod client; pub mod config; diff --git a/netmito/src/schema/agent.rs b/netmito/src/schema/agent.rs index c8636e45..57d606f6 100644 --- a/netmito/src/schema/agent.rs +++ b/netmito/src/schema/agent.rs @@ -59,6 +59,9 @@ pub struct RegisterAgentResp { pub notification_counter: u64, /// True if the machine was already known and its agent row is used again. pub reused: bool, + /// The Redis URL for the execution states of tasks. `None` if the coordinator has no Redis. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub redis_url: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/netmito/src/schema/exec.rs b/netmito/src/schema/exec.rs index 9de5b9a8..133bf961 100644 --- a/netmito/src/schema/exec.rs +++ b/netmito/src/schema/exec.rs @@ -36,7 +36,8 @@ pub struct ExecHooks { /// Environment cleanup hook (runs after suite completes) #[serde(default, skip_serializing_if = "Option::is_none")] pub cleanup: Option, - /// Background process hook (runs alongside workers) + /// Background process hook (runs alongside workers). The agent stops it with SIGTERM when the + /// tasks of the job end. The hook must then exit with 0, or its record is `Failed`. #[serde(default, skip_serializing_if = "Option::is_none")] pub background: Option, } diff --git a/netmito/src/schema/task.rs b/netmito/src/schema/task.rs index 0616e88c..cf5818c1 100644 --- a/netmito/src/schema/task.rs +++ b/netmito/src/schema/task.rs @@ -105,6 +105,8 @@ pub enum TaskResultMessage { // May record the user name who cancels the task. UserCancellation, SubmitNewTaskFailed, + /// The agent stopped the process, for example a background hook when the tasks of the job end. + StoppedByAgent, } #[derive(Debug, Serialize, Deserialize, FromQueryResult, Clone)] diff --git a/netmito/src/service/agent/mod.rs b/netmito/src/service/agent/mod.rs index 88d47f03..c51f4265 100644 --- a/netmito/src/service/agent/mod.rs +++ b/netmito/src/service/agent/mod.rs @@ -41,7 +41,7 @@ use sea_orm::sea_query::{Alias, PgFunc, Query}; use sea_orm::{prelude::*, FromQueryResult, QueryOrder, QuerySelect, Set, TransactionTrait}; use uuid::Uuid; -use crate::config::InfraPool; +use crate::config::{InfraPool, REDIS_CONNECTION_INFO}; use crate::entity::{ active_tasks as ActiveTasks, agents as Agent, group_agent as GroupAgent, groups as Group, machines as Machines, @@ -299,6 +299,8 @@ pub async fn user_register_agent( token, notification_counter, reused, + // Agents use the Redis account of the workers: both runners use the same task state keys. + redis_url: REDIS_CONNECTION_INFO.get().map(|info| info.worker_url()), }) } diff --git a/netmito/src/worker.rs b/netmito/src/worker.rs index a3a701c2..ec083e05 100644 --- a/netmito/src/worker.rs +++ b/netmito/src/worker.rs @@ -8,7 +8,9 @@ use tracing_subscriber::layer::SubscriberExt; use crate::config::TracingGuard; use crate::error::RequestError; -use crate::executor::{Executor, RESOURCE_FETCH_BUDGET, SINGLE_DOWNLOAD_BUDGET, UPLOAD_BUDGET}; +use crate::executor::{ + ExecKind, Executor, RESOURCE_FETCH_BUDGET, SINGLE_DOWNLOAD_BUDGET, UPLOAD_BUDGET, +}; use crate::schema::*; use crate::service::auth::get_and_prompt_username; use crate::{ @@ -121,13 +123,14 @@ impl MitoWorker { let resp: RegisterWorkerResp = resp.json().await.map_err(RequestError::from)?; let mut cache_path = dirs::cache_dir().ok_or(Error::Custom("Cache dir not found".to_string()))?; + crate::executor::warn_unreachable_ancestors(&cache_path).await; cache_path.push("mitosis"); + crate::executor::ensure_traversable_dir(&cache_path).await?; let log_dir = cache_path.join("worker"); cache_path.push(resp.worker_id.to_string()); - tokio::fs::create_dir_all(&cache_path).await?; - tokio::fs::create_dir_all(&cache_path.join("result")).await?; - tokio::fs::create_dir_all(&cache_path.join("exec")).await?; - tokio::fs::create_dir_all(&cache_path.join("resource")).await?; + // The executor makes the same tree again between tasks, with the same modes. Thus the + // first task also gets a working directory that does not use the umask. + crate::executor::reset_workspace(&cache_path).await?; tokio::fs::create_dir_all(&log_dir).await?; let guards = config.setup_tracing_subscriber::<&uuid::Uuid, _>(&resp.worker_id)?; // The worker's own subscriber is live now; stop shadowing it. @@ -185,9 +188,12 @@ impl MitoWorker { task_credential: self.credential.clone(), http_url: self.config.coordinator_addr.clone(), task_cancel_token: self.cancel_token.clone(), + task_stop_token: CancellationToken::new(), + runner_cancel_token: self.cancel_token.clone(), coordinator_force_exit: self.coordinator_force_exit.clone(), polling_interval: self.config.polling_interval, task_cache_path: self.cache_path.clone(), + extra_envs: Vec::new(), task_redis_conn, task_redis_pubsub, resource_fetch_budget: RESOURCE_FETCH_BUDGET, @@ -270,7 +276,10 @@ impl MitoWorker { if resp.status().is_success() { match resp.json::>().await { Ok(task) => match task { - Some(task) => match task_executor.execute(task).await { + Some(task) => match task_executor + .execute(ExecKind::Task { job: None }, task) + .await + { Ok(_) => {} Err(e) => { tracing::error!("Task execution failed: {}", e); diff --git a/openapi.yaml b/openapi.yaml index 1bdef393..c8622e3e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -3740,6 +3740,7 @@ components: - WatchTimeout - UserCancellation - SubmitNewTaskFailed + - StoppedByAgent description: | Task failure reason: - FetchResourceTimeout: Timeout downloading resources @@ -3750,6 +3751,7 @@ components: - WatchTimeout: Watched task timeout - UserCancellation: Task cancelled by user - SubmitNewTaskFailed: Failed to submit new task + - StoppedByAgent: The agent stopped the process (a background hook when the tasks of its job end) TasksQueryReq: type: object @@ -4885,7 +4887,10 @@ components: description: Hook run after the suite completes background: type: object - description: Hook run alongside workers + description: >- + Hook run alongside workers. The agent stops it with SIGTERM when the + tasks of the job end. The hook must then exit with 0, or its record is + Failed. CreateTaskSuiteReq: type: object @@ -5391,6 +5396,12 @@ components: reused: type: boolean description: True if the machine was already known and its agent record is used again + redis_url: + type: string + nullable: true + description: >- + The Redis URL for the execution states of tasks. Not present if the + coordinator has no Redis. AgentInfo: type: object diff --git a/src/main.rs b/src/main.rs index 7100479e..c7182c25 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,11 @@ use crate::build::CLAP_LONG_VERSION; use clap::{Parser, Subcommand}; use netmito::{ + agent::MitoAgent, client::MitoClient, - config::{ClientConfigCli, CoordinatorConfigCli, ManagerConfigCli, WorkerConfigCli}, + config::{ + AgentConfigCli, ClientConfigCli, CoordinatorConfigCli, ManagerConfigCli, WorkerConfigCli, + }, coordinator::MitoCoordinator, manager::MitoManager, worker::MitoWorker, @@ -26,6 +29,8 @@ enum Mode { Coordinator(CoordinatorConfigCli), /// Run a mitosis worker. Worker(WorkerConfigCli), + /// Run a mitosis agent, which executes task suites. + Agent(AgentConfigCli), /// Run a mitosis client. Client(ClientConfigCli), /// Manage mitosis workers. @@ -53,6 +58,17 @@ fn main() { MitoWorker::main(worker_cli).await; }); } + Mode::Agent(agent_cli) => { + // Multi-threaded: the agent runs its main loop, a WebSocket reader and the task slots + // of a job at the same time. + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + MitoAgent::main(agent_cli).await; + }); + } Mode::Client(client_cli) => { tokio::runtime::Builder::new_current_thread() .enable_all()