diff --git a/netmito/src/api/suites.rs b/netmito/src/api/suites.rs index 149cbb6..dec0314 100644 --- a/netmito/src/api/suites.rs +++ b/netmito/src/api/suites.rs @@ -11,8 +11,9 @@ use crate::{ config::InfraPool, error::ApiError, schema::{ - CancelTaskSuiteParam, CreateTaskSuiteReq, CreateTaskSuiteResp, SuiteAgentOverrideReq, - SuiteAgentOverrideResp, TaskSuiteQueryResp, TaskSuitesQueryReq, TaskSuitesQueryResp, + CancelTaskSuiteParam, ChangeTaskSuiteReq, CreateTaskSuiteReq, CreateTaskSuiteResp, + SuiteAgentOverrideReq, SuiteAgentOverrideResp, TaskSuiteQueryResp, TaskSuitesQueryReq, + TaskSuitesQueryResp, }, service::{ self, @@ -24,7 +25,12 @@ pub fn suites_router(st: InfraPool) -> Router { Router::new() .route("/", post(create_suite)) .route("/query", post(query_suites)) - .route("/{uuid}", get(get_suite_details).delete(cancel_suite)) + .route( + "/{uuid}", + get(get_suite_details) + .put(change_suite) + .delete(cancel_suite), + ) .route("/{uuid}/close", post(close_suite)) .route("/{uuid}/agents/override", post(override_agents_for_suite)) .route_layer(middleware::from_fn_with_state( @@ -67,6 +73,18 @@ pub async fn get_suite_details( Ok(Json(details)) } +pub async fn change_suite( + Extension(u): Extension, + State(pool): State, + Path(uuid): Path, + Json(req): Json, +) -> Result<(), ApiError> { + service::suite::user_change_task_suite(u.id, &pool, uuid, req) + .await + .map_err(map_service_error)?; + Ok(()) +} + pub async fn close_suite( Extension(u): Extension, State(pool): State, diff --git a/netmito/src/client/http.rs b/netmito/src/client/http.rs index 9627780..94e0299 100644 --- a/netmito/src/client/http.rs +++ b/netmito/src/client/http.rs @@ -1680,6 +1680,32 @@ impl MitoHttpClient { } } + pub async fn change_task_suite( + &mut self, + uuid: Uuid, + req: ChangeTaskSuiteReq, + ) -> crate::error::Result<()> { + let credential = self + .credential_guard + .get_credential() + .ok_or(CredentialGuardError::CredentialNotFound)? + .token; + self.url.set_path(&format!("suites/{uuid}")); + let resp = self + .http_client + .put(self.url.as_str()) + .bearer_auth(credential) + .json(&req) + .send() + .await + .map_err(map_reqwest_err)?; + if resp.status().is_success() { + Ok(()) + } else { + Err(get_error_from_resp(resp).await.into()) + } + } + pub async fn close_task_suite(&mut self, uuid: Uuid) -> crate::error::Result<()> { let credential = self .credential_guard diff --git a/netmito/src/client/interactive.rs b/netmito/src/client/interactive.rs index d3e8898..c017a8b 100644 --- a/netmito/src/client/interactive.rs +++ b/netmito/src/client/interactive.rs @@ -111,6 +111,9 @@ pub(crate) fn output_parsed_task_info(info: &ParsedTaskQueryInfo) { if let Some(downstream_task_uuid) = info.downstream_task_uuid { tracing::info!("Downstream Task UUID: {:?}", downstream_task_uuid); } + if let Some(suite_uuid) = info.suite_uuid { + tracing::info!("Task Suite UUID: {}", suite_uuid); + } if let Some(runner_uuid) = info.runner_uuid { tracing::info!("Runner UUID: {}", runner_uuid); } @@ -149,6 +152,9 @@ pub(crate) fn output_task_info(info: &TaskQueryInfo) { if let Some(downstream_task_uuid) = info.downstream_task_uuid { tracing::info!("Downstream Task UUID: {:?}", downstream_task_uuid); } + if let Some(suite_uuid) = info.suite_uuid { + tracing::info!("Task Suite UUID: {}", suite_uuid); + } if let Some(runner_uuid) = info.runner_uuid { tracing::info!("Runner UUID: {}", runner_uuid); } else { diff --git a/netmito/src/client/mod.rs b/netmito/src/client/mod.rs index b818005..98ccd76 100644 --- a/netmito/src/client/mod.rs +++ b/netmito/src/client/mod.rs @@ -820,6 +820,12 @@ impl MitoClient { self.http_client.create_task_suite(args.into()).await } + pub async fn suites_change(&mut self, args: ChangeSuiteArgs) -> crate::error::Result<()> { + self.http_client + .change_task_suite(args.uuid, args.into()) + .await + } + pub async fn suites_query( &mut self, args: QuerySuitesArgs, @@ -1879,8 +1885,19 @@ impl MitoClient { output_task_info(&task); } } else { + // Without --verbose, the output shows only the task UUID. + // For a task in a suite, it also shows the suite UUID. for task in resp.tasks { - tracing::info!("{}", task.uuid); + match task.suite_uuid { + Some(suite_uuid) => { + tracing::info!( + "{}, suite {}", + task.uuid, + suite_uuid + ) + } + None => tracing::info!("{}", task.uuid), + } } } } @@ -2051,7 +2068,7 @@ impl MitoClient { }, }, ClientCommand::Suites(args) => match args.command { - SuitesCommands::Create(args) => match self.suites_create(args).await { + SuitesCommands::Create(args) => match self.suites_create(*args).await { Ok(resp) => { tracing::info!("Suite created with uuid {}", resp.uuid); } @@ -2059,6 +2076,14 @@ impl MitoClient { tracing::error!("{}", e); } }, + SuitesCommands::Change(args) => match self.suites_change(*args).await { + Ok(_) => { + tracing::info!("Suite changed successfully"); + } + Err(e) => { + tracing::error!("{}", e); + } + }, SuitesCommands::Query(args) => { let verbose = args.verbose; let counted = args.count; diff --git a/netmito/src/config/client/artifacts.rs b/netmito/src/config/client/artifacts.rs index 5766d08..bf5600d 100644 --- a/netmito/src/config/client/artifacts.rs +++ b/netmito/src/config/client/artifacts.rs @@ -88,6 +88,10 @@ pub struct DownloadArtifactsByFilterArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -152,6 +156,7 @@ impl From for ArtifactsDownloadByFilterReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { @@ -194,6 +199,10 @@ pub struct DeleteArtifactsByFilterArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -234,6 +243,7 @@ impl From for ArtifactsDeleteByFilterReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { diff --git a/netmito/src/config/client/mod.rs b/netmito/src/config/client/mod.rs index 0d22191..50485fc 100644 --- a/netmito/src/config/client/mod.rs +++ b/netmito/src/config/client/mod.rs @@ -9,7 +9,7 @@ use std::ops::Not; use url::Url; use crate::entity::content::ArtifactContentType; -use crate::schema::{RemoteResource, RemoteResourceDownload}; +use crate::schema::{ExecSpec, RemoteResource, RemoteResourceDownload}; use std::path::PathBuf; use uuid::Uuid; @@ -158,6 +158,20 @@ fn parse_artifact_content_type( } } +/// Parse a hook `ExecSpec` from a JSON object. The object has the same shape that `POST /suites` +/// accepts, for example `{"args":["sh","-c","./setup.sh"],"terminal_output":true}`. +/// Only `args` is necessary. +fn parse_exec_spec( + s: &str, +) -> Result> { + let spec: ExecSpec = serde_json::from_str(s) + .map_err(|e| format!("invalid hook spec: {e}. Expected a JSON object such as {{\"args\":[\"sh\",\"-c\",\"./setup.sh\"],\"terminal_output\":true}}"))?; + if spec.args.is_empty() { + return Err("invalid hook spec: `args` must not be empty".into()); + } + Ok(spec) +} + /// Parse a resource string into RemoteResourceDownload fn parse_resources( s: &str, diff --git a/netmito/src/config/client/suites.rs b/netmito/src/config/client/suites.rs index e18ddc4..cd3a08c 100644 --- a/netmito/src/config/client/suites.rs +++ b/netmito/src/config/client/suites.rs @@ -1,12 +1,19 @@ +use std::collections::HashSet; + use clap::{Args, Subcommand}; use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::{ entity::state::TaskSuiteState, - schema::{CreateTaskSuiteReq, TaskSuitesQueryReq, WorkerSchedulePlan}, + schema::{ + ChangeTaskSuiteReq, CreateTaskSuiteReq, ExecHooks, ExecHooksUpdate, ExecSpec, + TaskSuitesQueryReq, UpdateOp, WorkerSchedulePlan, + }, }; +use super::parse_exec_spec; + #[derive(Serialize, Debug, Deserialize, Args, derive_more::From, Clone)] pub struct SuitesArgs { #[command(subcommand)] @@ -16,7 +23,11 @@ pub struct SuitesArgs { #[derive(Subcommand, Serialize, Debug, Deserialize, Clone)] pub enum SuitesCommands { /// Create a new task suite - Create(CreateSuiteArgs), + // Put the arguments in a box: the three optional hook specs make this variant much larger + // than the other variants. + Create(Box), + /// Change the properties of a suite. The change applies to the jobs that start after it + Change(Box), /// Query task suites subject to a filter Query(QuerySuitesArgs), /// Get the details of a task suite @@ -55,10 +66,32 @@ pub struct CreateSuiteArgs { /// Number of tasks each worker prefetches locally #[arg(long, default_value_t = 16)] pub prefetch: u32, + /// The provision hook, as a JSON exec spec, for example + /// '{"args":["sh","-c","./setup.sh"],"terminal_output":true}'. It runs one time before the + /// tasks. If it exits with a non-zero code, the job fails and its tasks do not start + #[arg(long, value_parser = parse_exec_spec)] + pub provision: Option, + /// The cleanup hook, with the same JSON shape. It runs one time after the last task of the + /// job ends. It also runs if the job failed + #[arg(long, value_parser = parse_exec_spec)] + pub cleanup: Option, + /// The background hook, with the same JSON shape. It runs at the same time as the tasks, and + /// usually continues to run until the job ends + #[arg(long, value_parser = parse_exec_spec)] + pub background: Option, } impl From for CreateTaskSuiteReq { fn from(args: CreateSuiteArgs) -> Self { + // If no hook is given, keep `exec_hooks` absent. Do not send an object with three nulls. + let exec_hooks = match (args.provision, args.cleanup, args.background) { + (None, None, None) => None, + (provision, cleanup, background) => Some(ExecHooks { + provision, + cleanup, + background, + }), + }; Self { name: args.name, description: args.description, @@ -71,7 +104,121 @@ impl From for CreateTaskSuiteReq { cpu_binding: None, task_prefetch_count: args.prefetch, }, - exec_hooks: None, + exec_hooks, + } + } +} + +/// A field does not change if you do not give its flag. Each nullable field has a `--clear-*` +/// flag that conflicts with its value flag. If you set and clear the same field, clap shows a +/// parse error, and the client does not send the request. +#[derive(Serialize, Debug, Deserialize, Args, Clone)] +pub struct ChangeSuiteArgs { + /// The UUID of the suite + pub uuid: Uuid, + /// The new name of the suite + #[arg(short, long, conflicts_with = "clear_name")] + pub name: Option, + /// Remove the name of the suite + #[arg(long)] + pub clear_name: bool, + /// The new description of the suite + #[arg(short, long, conflicts_with = "clear_description")] + pub description: Option, + /// Remove the description of the suite + #[arg(long)] + pub clear_description: bool, + /// The new tags that an agent must have to run the suite (for example, gpu,linux) + #[arg(short, long, num_args = 1.., value_delimiter = ',', conflicts_with = "clear_tags")] + pub tags: Option>, + /// Remove all tags. A suite with no tags matches all agents that its group can write to + #[arg(long)] + pub clear_tags: bool, + /// The new labels, which you can use to query and filter suites + #[arg(short, long, num_args = 1.., value_delimiter = ',', conflicts_with = "clear_labels")] + pub labels: Option>, + /// Remove all labels + #[arg(long)] + pub clear_labels: bool, + /// The scheduling priority of the suite. A higher value is more important + #[arg(short, long)] + pub priority: Option, + /// The number of workers that each agent starts for this suite. You must also give + /// `--prefetch` + #[arg(short, long, requires = "prefetch")] + pub workers: Option, + /// The number of tasks that each worker gets before it needs them. You must also give + /// `--workers` + #[arg(long, requires = "workers")] + pub prefetch: Option, + /// The new provision hook, as a JSON exec spec + #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_provision")] + pub provision: Option, + /// Remove the provision hook + #[arg(long)] + pub clear_provision: bool, + /// The new cleanup hook, with the same JSON shape + #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_cleanup")] + pub cleanup: Option, + /// Remove the cleanup hook + #[arg(long)] + pub clear_cleanup: bool, + /// The new background hook, with the same JSON shape + #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_background")] + pub background: Option, + /// Remove the background hook + #[arg(long)] + pub clear_background: bool, +} + +/// Convert a value flag and its `--clear-*` flag into one `UpdateOp`. clap does not accept the +/// two flags together. If the two flags arrive, the value is used. +fn update_op(value: Option, clear: bool) -> Option> { + match (value, clear) { + (Some(value), _) => Some(UpdateOp::SetTo(value)), + (None, true) => Some(UpdateOp::SetNull), + (None, false) => None, + } +} + +impl From for ChangeTaskSuiteReq { + fn from(args: ChangeSuiteArgs) -> Self { + // An array column cannot be null, so its cleared value is an empty set. The CLI uses the + // same flag pair for all fields, but the request is different for these two fields. + let tags = match (args.tags, args.clear_tags) { + (Some(tags), _) => Some(tags.into_iter().collect()), + (None, true) => Some(HashSet::new()), + (None, false) => None, + }; + let labels = match (args.labels, args.clear_labels) { + (Some(labels), _) => Some(labels.into_iter().collect()), + (None, true) => Some(HashSet::new()), + (None, false) => None, + }; + // The request replaces the full plan, so it must contain the full plan or no plan. clap + // requires the two flags together. A missing flag must not get a default value that + // replaces the current value of the suite. + let worker_schedule = match (args.workers, args.prefetch) { + (Some(worker_count), Some(prefetch)) => Some(WorkerSchedulePlan::FixedWorkers { + worker_count, + cpu_binding: None, + task_prefetch_count: prefetch, + }), + _ => None, + }; + let hooks = ExecHooksUpdate { + provision: update_op(args.provision, args.clear_provision), + cleanup: update_op(args.cleanup, args.clear_cleanup), + background: update_op(args.background, args.clear_background), + }; + Self { + name: update_op(args.name, args.clear_name), + description: update_op(args.description, args.clear_description), + tags, + labels, + priority: args.priority, + worker_schedule, + exec_hooks: (!hooks.is_empty()).then_some(hooks), } } } diff --git a/netmito/src/config/client/tasks.rs b/netmito/src/config/client/tasks.rs index 5d45010..7023ebe 100644 --- a/netmito/src/config/client/tasks.rs +++ b/netmito/src/config/client/tasks.rs @@ -96,6 +96,10 @@ pub struct QueryTasksArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -142,6 +146,10 @@ pub struct CancelTasksArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -215,6 +223,7 @@ impl From for TasksQueryReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { @@ -283,6 +292,7 @@ impl From for TasksCancelByFilterReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { diff --git a/netmito/src/schema/artifact.rs b/netmito/src/schema/artifact.rs index 47f153d..c8fba3f 100644 --- a/netmito/src/schema/artifact.rs +++ b/netmito/src/schema/artifact.rs @@ -32,6 +32,7 @@ pub struct ArtifactQueryResp { pub struct ArtifactsDownloadByFilterReq { pub creator_usernames: Option>, pub group_name: Option, + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, @@ -70,6 +71,7 @@ pub struct ArtifactsDownloadListResp { pub struct ArtifactsDeleteByFilterReq { pub creator_usernames: Option>, pub group_name: Option, + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, diff --git a/netmito/src/schema/suite.rs b/netmito/src/schema/suite.rs index d97a916..c43654c 100644 --- a/netmito/src/schema/suite.rs +++ b/netmito/src/schema/suite.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use crate::entity::state::TaskSuiteState; -use super::exec::ExecHooks; +use super::exec::{ExecHooks, ExecSpec}; /// Request to create a new task suite #[derive(Debug, Clone, Serialize, Deserialize)] @@ -43,6 +43,96 @@ pub struct CreateTaskSuiteResp { pub uuid: Uuid, } +/// A change to a field that has a nullable column. The request puts the field in an `Option`, +/// and an absent field does not change. +/// A present field has one of these two values: set a new value, or set the column to null. +/// +/// A field that has a `NOT NULL` column uses a plain `Option`. This column cannot be null, so its +/// 'cleared' value is a value of its type, for example an empty array. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "op", content = "value", rename_all = "snake_case")] +pub enum UpdateOp { + SetTo(T), + SetNull, +} + +impl UpdateOp { + /// Returns the value to store. `SetNull` returns `None`. + pub fn into_value(self) -> Option { + match self { + Self::SetTo(value) => Some(value), + Self::SetNull => None, + } + } +} + +/// The changes to the `exec_hooks` of a suite, one for each hook. You can set a hook, clear a +/// hook, or leave a hook out to keep it. The request does not replace the full object, because +/// that would remove each hook that the request does not mention. If you clear all three hooks, +/// the suite stores NULL, the same as a suite that you create without hooks. +/// +/// The database merges the changes into the current value of the column. Because of this, two +/// changes at the same time cannot overwrite each other. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ExecHooksUpdate { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provision: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cleanup: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub background: Option>, +} + +impl ExecHooksUpdate { + pub fn is_empty(&self) -> bool { + self.provision.is_none() && self.cleanup.is_none() && self.background.is_none() + } +} + +/// A request to change the properties of a suite. An absent field means no change. +/// The server rejects an empty request. +/// +/// The change applies to the jobs that start after it, because a job copies the worker schedule +/// and the hooks when it starts. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ChangeTaskSuiteReq { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option>, + /// The tags that an agent must have to run the suite. An empty set matches all agents that + /// the group of the suite can write to. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub priority: Option, + /// Replaces the full plan. A partial change cannot change the plan to a different scheduling policy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_schedule: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exec_hooks: Option, +} + +impl ChangeTaskSuiteReq { + pub fn is_empty(&self) -> bool { + self.name.is_none() + && self.description.is_none() + && self.tags.is_none() + && self.labels.is_none() + && self.priority.is_none() + && self.worker_schedule.is_none() + && self.exec_hooks.as_ref().map_or(true, |h| h.is_empty()) + } + + /// Returns true if the change can change the result of the scheduler: which agents can run the + /// suite, or the position of the suite in the pick order. + pub fn affects_scheduling(&self) -> bool { + self.tags.is_some() || self.priority.is_some() + } +} + /// Worker scheduling policy for the suite /// This enum allows for future extension with different scheduling strategies #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/netmito/src/schema/task.rs b/netmito/src/schema/task.rs index 56a73cf..89b29d3 100644 --- a/netmito/src/schema/task.rs +++ b/netmito/src/schema/task.rs @@ -100,6 +100,8 @@ pub struct TaskQueryInfo { pub uuid: Uuid, pub creator_username: String, pub group_name: String, + /// The suite of this task, if the task is in a suite. The `group_name` above must own this suite. + pub suite_uuid: Option, pub task_id: i64, pub tags: Vec, pub labels: Vec, @@ -120,6 +122,7 @@ pub struct ParsedTaskQueryInfo { pub uuid: Uuid, pub creator_username: String, pub group_name: String, + pub suite_uuid: Option, pub task_id: i64, pub tags: Vec, pub labels: Vec, @@ -142,6 +145,10 @@ pub struct ParsedTaskQueryInfo { pub struct TasksQueryReq { pub creator_usernames: Option>, pub group_name: Option, + /// Show only the tasks of this suite. The suite also identifies the group. If `group_name` + /// is absent, the server uses the group of the suite. If `group_name` is a different group, + /// the server returns 404, the same as for a suite that does not exist. + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, @@ -160,6 +167,7 @@ pub struct TasksQueryReq { pub struct TasksCancelByFilterReq { pub creator_usernames: Option>, pub group_name: Option, + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, diff --git a/netmito/src/service/mod.rs b/netmito/src/service/mod.rs index c7b4edd..f3229fa 100644 --- a/netmito/src/service/mod.rs +++ b/netmito/src/service/mod.rs @@ -1,3 +1,25 @@ +//! # Lock order: suite before task +//! +//! A transaction that writes a `task_suites` row and `active_tasks` rows must write the suite +//! row first. +//! +//! An `UPDATE` or `DELETE` keeps its row locks until the transaction commits, not until the +//! statement ends. Thus, a write in one statement does not solve the order problem. A +//! transaction keeps all of its locks. If two transactions lock the same two rows in opposite +//! order, they deadlock. Postgres finds the deadlock only after `deadlock_timeout`. +//! +//! There are two correct ways to keep this order: +//! +//! - The suite id is known before the write, for example on each commit and on a single-task +//! cancel. Write the suite row first, then the task rows. +//! - The suite ids come only from the task rows, for example on a batch cancel. Lock the suites +//! in the same statement that removes the tasks, and remove a task only after its suite is +//! locked. +//! +//! This rule does not apply to a transaction that locks task rows and does not write a suite +//! row. `claim_candidates` locks only task rows with `FOR UPDATE SKIP LOCKED`, and no +//! transaction waits for these locks. + pub mod auth; pub mod group; pub mod s3; diff --git a/netmito/src/service/s3.rs b/netmito/src/service/s3.rs index e56b951..5221f51 100644 --- a/netmito/src/service/s3.rs +++ b/netmito/src/service/s3.rs @@ -253,6 +253,7 @@ pub async fn batch_download_artifacts_by_filter( let query = TasksQueryReq { creator_usernames: req.creator_usernames, group_name: req.group_name, + suite_uuid: req.suite_uuid, tags: req.tags, labels: req.labels, states: req.states, @@ -1322,6 +1323,7 @@ pub async fn batch_delete_artifacts_by_filter( let query = TasksQueryReq { creator_usernames: req.creator_usernames, group_name: req.group_name, + suite_uuid: req.suite_uuid, tags: req.tags, labels: req.labels, states: req.states, diff --git a/netmito/src/service/suite.rs b/netmito/src/service/suite.rs index d68ae44..8407de6 100644 --- a/netmito/src/service/suite.rs +++ b/netmito/src/service/suite.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use sea_orm::sea_query::extension::postgres::PgExpr; -use sea_orm::sea_query::{Alias, PgFunc, Query}; +use sea_orm::sea_query::{ + Alias, CommonTableExpression, DeleteStatement, InsertStatement, PgFunc, Query, WithClause, +}; use sea_orm::{prelude::*, ConnectionTrait, FromQueryResult, Set, TransactionTrait}; use uuid::Uuid; @@ -17,18 +19,125 @@ use crate::entity::{ }; use crate::error::{ApiError, Error, ResolveError, Result}; use crate::schema::{ - CancelTaskSuiteOp, CountQuery, CreateTaskSuiteReq, CreateTaskSuiteResp, ExecHooks, - ParsedTaskSuiteInfo, SuiteAgentOverrideReq, SuiteAgentOverrideResp, TaskResultMessage, - TaskResultSpec, TaskSuiteInfo, TaskSuiteQueryResp, TaskSuitesQueryReq, TaskSuitesQueryResp, - WorkerSchedulePlan, + CancelTaskSuiteOp, ChangeTaskSuiteReq, CountQuery, CreateTaskSuiteReq, CreateTaskSuiteResp, + ExecHooks, ParsedTaskSuiteInfo, SuiteAgentOverrideReq, SuiteAgentOverrideResp, + TaskResultMessage, TaskResultSpec, TaskSuiteInfo, TaskSuiteQueryResp, TaskSuitesQueryReq, + TaskSuitesQueryResp, UpdateOp, WorkerSchedulePlan, }; -use crate::service::task::{parse_operators_with_number, OperatorWithNumber}; +use crate::service::task::{check_exec_spec, parse_operators_with_number, OperatorWithNumber}; #[derive(FromQueryResult)] struct GroupIdResult { id: i64, } +#[derive(FromQueryResult)] +struct TaskUuidResult { + uuid: Uuid, +} + +/// Decrease `incomplete_tasks` of a suite by `count`, because `count` of its tasks are finished. +/// If this makes the counter of a `Closed` suite zero, the suite becomes `Complete`. +/// +/// Each path that removes an active task from a suite must call this function. If a path archives +/// a suite task and does not call it, the suite cannot become `Complete`. Agents then continue to +/// look for tasks that do not exist. +/// +/// The subtraction has no lower limit, and this is intentional. Each caller removed exactly `count` +/// rows from `active_tasks` immediately before the call. Each of these rows increased the counter +/// by one when it was submitted. Thus, the counter cannot correctly go below zero. A lower limit +/// would hide a bug in a different part of the code. Instead, the function logs an error for a +/// negative counter. +/// +/// When an `Open` suite has no more tasks, it does not become `Complete` here. It stays `Open` +/// until it also gets no new task for `suite_auto_close_timeout`. During this time, the agent can +/// keep its job and its provisioned environment for a task that arrives late. Then +/// [`sweep_inactive_suites`] changes the state of the suite. A `Closed` suite does not wait, so +/// this function makes it `Complete` immediately. +/// +/// The suite row must be locked before all task rows, by this call or by the caller before it. +/// Refer to the lock-order note on [`crate::service`]. +pub(crate) async fn decrement_incomplete_tasks( + db: &C, + suite_id: i64, + count: i32, + now: TimeDateTimeWithTimeZone, +) -> Result<()> { + if count <= 0 { + return Ok(()); + } + + let updated = TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::IncompleteTasks, + Expr::col(TaskSuites::Column::IncompleteTasks).sub(count), + ) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .filter(TaskSuites::Column::Id.eq(suite_id)) + .exec_with_returning(db) + .await?; + + // This function cannot repair a negative counter. It does not set the counter to zero, so + // that the bug is easier to find. + for suite in updated.iter().filter(|s| s.incomplete_tasks < 0) { + tracing::error!( + suite_id = suite.id, + incomplete_tasks = suite.incomplete_tasks, + decremented = count, + "A suite's incomplete_tasks went negative: more tasks were taken off it than were ever counted" + ); + } + + // This statement has conditions on the state and on the counter. Because of this, it is safe + // as a second statement without a transaction around the two statements. A submit that occurs + // at the same time either comes first and makes the counter more than zero, or comes after and + // sets the state to `Open` itself. + TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Complete), + ) + .col_expr(TaskSuites::Column::CompletedAt, Expr::value(Some(now))) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .filter(TaskSuites::Column::Id.eq(suite_id)) + .filter(TaskSuites::Column::State.eq(TaskSuiteState::Closed)) + .filter(TaskSuites::Column::IncompleteTasks.eq(0)) + .exec(db) + .await?; + + Ok(()) +} + +/// The maximum number of workers that a plan can request from an agent. +const MAX_WORKER_COUNT: u32 = 256; + +/// Make sure that the parameters of a plan are in the permitted range. The create path and the +/// change path both use this function. +// TODO: change this check when the plan has a more flexible definition. +fn validate_worker_schedule(plan: &WorkerSchedulePlan) -> Result<()> { + match plan { + WorkerSchedulePlan::FixedWorkers { worker_count, .. } => { + if *worker_count == 0 || *worker_count > MAX_WORKER_COUNT { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "worker_count must be between 1 and {MAX_WORKER_COUNT}, got {worker_count}" + )))); + } + } + } + Ok(()) +} + +/// Hooks use the same executor as tasks, so their specs get the same check as task specs. +fn validate_exec_hooks(hooks: &ExecHooks) -> Result<()> { + for spec in [&hooks.provision, &hooks.cleanup, &hooks.background] + .into_iter() + .flatten() + { + check_exec_spec(spec)?; + } + Ok(()) +} + pub async fn user_create_task_suite( user_id: i64, pool: &InfraPool, @@ -43,16 +152,9 @@ pub async fn user_create_task_suite( exec_hooks, }: CreateTaskSuiteReq, ) -> crate::error::Result { - // Validate the worker schedule based on the policy variant. - match &worker_schedule { - // TODO: this should finally be adjusted to some more flexible definitions - WorkerSchedulePlan::FixedWorkers { - worker_count: _worker_count, - task_prefetch_count: _task_prefetch_count, - .. - } => { - // TODO: validate the request - } + validate_worker_schedule(&worker_schedule)?; + if let Some(hooks) = &exec_hooks { + validate_exec_hooks(hooks)?; } if group_name.is_empty() { @@ -120,9 +222,8 @@ pub async fn user_create_task_suite( /// Validate a suite query and resolve/authorize its target group. /// -/// Rejects empty-but-present filter sets, defaults `group_name` to the caller's -/// username when omitted, and requires the caller to have at least `Read` in the -/// resolved group. +/// The function rejects a filter set that is present and empty. If `group_name` is absent, it +/// uses the username of the caller. The caller must have at least `Read` in the group. pub(crate) async fn check_task_suites_query( user_id: i64, pool: &InfraPool, @@ -211,6 +312,159 @@ pub(crate) async fn check_task_suites_query( Ok(()) } +/// Change the properties of a suite. The caller must have `Write` in the group of the suite. +/// +/// You cannot change a `Cancelled` suite. You can change a suite in all other states. +/// +/// The change applies to the jobs that start after it, because a job copies the worker schedule +/// and the hooks when it starts. +pub async fn user_change_task_suite( + user_id: i64, + pool: &InfraPool, + suite_uuid: Uuid, + req: ChangeTaskSuiteReq, +) -> Result<()> { + if req.is_empty() { + return Err(Error::ApiError(ApiError::InvalidRequest( + "No change specified".to_string(), + ))); + } + if let Some(plan) = &req.worker_schedule { + validate_worker_schedule(plan)?; + } + // Check only the new specs. The server checked the other hooks when it wrote them. + if let Some(update) = &req.exec_hooks { + for op in [&update.provision, &update.cleanup, &update.background] + .into_iter() + .flatten() + { + if let UpdateOp::SetTo(spec) = op { + check_exec_spec(spec)?; + } + } + } + // If the change has new tags or a new priority, tell the agents that the suite is available. + // New tags can let an idle agent run this suite. A new priority can change which suite an idle + // agent selects or preempt an active agent. + let _needs_notify = req.affects_scheduling(); + let now = TimeDateTimeWithTimeZone::now_utc(); + + let _suite_id = pool + .db + .transaction::<_, i64, Error>(|txn| { + Box::pin(async move { + let suite = + match authorize_suite(txn, user_id, suite_uuid, UserGroupRole::Write).await { + Ok(suite) => suite, + Err(ResolveError::Item(e)) => { + return Err(Error::ApiError(ApiError::NotFound(e.msg))) + } + Err(ResolveError::Fatal(e)) => return Err(e), + }; + let suite_id = suite.id; + + // `authorize_suite` does not lock the row, so this check only puts the state in + // the error message. The `WHERE` clause below rejects a suite that a different + // request cancels after this check. + if suite.state == TaskSuiteState::Cancelled { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Cannot change a {} suite", + suite.state + )))); + } + + let mut change = TaskSuites::ActiveModel { + updated_at: Set(now), + ..Default::default() + }; + if let Some(op) = req.name { + change.name = Set(op.into_value()); + } + if let Some(op) = req.description { + change.description = Set(op.into_value()); + } + if let Some(tags) = req.tags { + change.tags = Set(Vec::from_iter(tags)); + } + if let Some(labels) = req.labels { + change.labels = Set(Vec::from_iter(labels)); + } + if let Some(priority) = req.priority { + change.priority = Set(priority); + } + if let Some(plan) = req.worker_schedule { + change.worker_schedule = Set(serde_json::to_value(plan)?); + } + // The request can set, clear or keep each hook, so the new value depends on the + // current value. If the code reads the row and merges in Rust, two changes at the + // same time can overwrite the hooks of each other. Thus, the column gets an SQL + // expression of its current value, and the merge occurs in the `UPDATE`. + let hooks_expr = match req.exec_hooks { + Some(update) => { + // A suite with no hooks stores NULL. Use `{}` for NULL, so that the + // first hook can be added. + let mut hooks = Expr::cust_with_expr( + "coalesce($1, '{}'::jsonb)", + Expr::col(TaskSuites::Column::ExecHooks), + ); + for (key, op) in [ + ("provision", update.provision), + ("cleanup", update.cleanup), + ("background", update.background), + ] { + hooks = match op { + // `||` sets this key and keeps the other keys. `-` removes it. + // + // Each step has parentheses, because `-` has a higher precedence + // than `||`. Without parentheses, PostgreSQL reads the chain as + // `x || (obj - 'cleanup')`. + Some(UpdateOp::SetTo(spec)) => Expr::cust_with_exprs( + "($1 || jsonb_build_object($2::text, $3::jsonb))", + [ + hooks, + Expr::val(key).into(), + Expr::val(serde_json::to_value(spec)?).into(), + ], + ), + Some(UpdateOp::SetNull) => Expr::cust_with_exprs( + "($1 - $2::text)", + [hooks, Expr::val(key).into()], + ), + None => hooks, + }; + } + // If no hook stays, store NULL, not an empty object. + Some(Expr::cust_with_expr("nullif($1, '{}'::jsonb)", hooks)) + } + None => None, + }; + + // `exec_hooks` is not in `change`, so this statement sets it only one time. + let mut stmt = TaskSuites::Entity::update_many() + .set(change) + .filter(TaskSuites::Column::Id.eq(suite_id)) + .filter(TaskSuites::Column::State.ne(TaskSuiteState::Cancelled)); + if let Some(expr) = hooks_expr { + stmt = stmt.col_expr(TaskSuites::Column::ExecHooks, expr); + } + let changed = stmt.exec(txn).await?; + if changed.rows_affected == 0 { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Suite {suite_uuid} was cancelled" + )))); + } + Ok(suite_id) + }) + }) + .await?; + + // Each query reads the eligibility and the pick order from the row, so there is nothing to + // rebuild. A notified agent does a new fetch and gets the latest values. + // TODO: once agents can be notified, `if _needs_notify` call + // `notify_suite_available(pool, _suite_id)` here. + Ok(()) +} + /// Query suites subject to a filter. Only returns suites in a group the caller can read. pub async fn user_query_task_suites( user_id: i64, @@ -330,6 +584,16 @@ pub async fn user_query_task_suites( } } } + if !query.count { + stmt.order_by_expr( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Priority)).into(), + sea_orm::Order::Desc, + ) + .order_by_expr( + Expr::col((TaskSuites::Entity, TaskSuites::Column::CreatedAt)).into(), + sea_orm::Order::Asc, + ); + } if let Some(limit) = query.limit { stmt.limit(limit); } @@ -388,11 +652,11 @@ struct AgentUuidResult { uuid: Uuid, } -/// Get a single suite's details, including the uuids of agents allowed to execute this suite. -/// The caller must have at least `Read` in the suite's group. +/// Get the details of one suite, with the UUIDs of the agents that can run it. The caller must +/// have at least `Read` in the group of the suite. /// -/// Currently we only report agents that are UserIncluded to the suite. In the future we -/// shall read the in-memory list for actual tag matching results. +/// At this time, the result shows only the agents that a user included in the suite. +/// TODO: read the in-memory list to show the agents that match the tags. pub async fn user_get_task_suite_by_uuid( user_id: i64, pool: &InfraPool, @@ -536,6 +800,8 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U Err(ResolveError::Fatal(e)) => return Err(e), }; + // `authorize_suite` does not lock the row, so this check only puts the state in + // the error message. The `WHERE` clause below rejects a second close. if suite.state != TaskSuiteState::Open { return Err(Error::ApiError(ApiError::InvalidRequest(format!( "Cannot transition from {} to Closed", @@ -543,10 +809,21 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U )))); } - let mut suite: TaskSuites::ActiveModel = suite.into(); - suite.state = Set(TaskSuiteState::Closed); - suite.updated_at = Set(now); - suite.update(txn).await?; + let closed = TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Closed), + ) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .filter(TaskSuites::Column::Id.eq(suite.id)) + .filter(TaskSuites::Column::State.eq(TaskSuiteState::Open)) + .exec(txn) + .await?; + if closed.rows_affected == 0 { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Suite {suite_uuid} is no longer Open" + )))); + } Ok(()) }) }) @@ -555,13 +832,16 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U Ok(()) } -/// Cancel a suite (`* → Cancelled`, terminal). Archives every non-terminal task -/// of the suite as `Cancelled`. Requires Write/Admin in the suite's group. +/// Cancel a suite (`* → Cancelled`). `Cancelled` is a terminal state. The function archives all +/// tasks of the suite that are not in a terminal state, with the state `Cancelled`. The caller +/// must have `Write` or `Admin` in the group of the suite. /// -/// `op` (`Graceful`/`Force`) currently has no differentiated effect: its only -/// distinction is how running agents/jobs tear down and get notified, which lives -/// in the agent layer that is not ported yet. The parameter is accepted now so the -/// API stays stable; wire its behavior in with the agent work (see the seams below). +/// `op` sets what occurs to the jobs that run the suite: +/// - `Graceful`: the server tells each agent that the suite is cancelled, and which of its tasks +/// are cancelled. The agent stops taking tasks, runs the cleanup hook, and ends the job as +/// `Completed`. +/// - `Force`: the server marks the jobs `Killed` immediately, and the cleanup hook does not run. +/// The server tells the agents to stop. pub async fn user_cancel_task_suite( user_id: i64, pool: &InfraPool, @@ -569,9 +849,11 @@ pub async fn user_cancel_task_suite( _op: CancelTaskSuiteOp, ) -> Result<()> { let now = TimeDateTimeWithTimeZone::now_utc(); + let builder = pool.db.get_database_backend(); - pool.db - .transaction::<_, u64, Error>(|txn| { + let (_suite_id, _cancelled_task_uuids) = pool + .db + .transaction::<_, (i64, Vec), Error>(|txn| { Box::pin(async move { // Resolve the suite and authorize the caller's Write role in one join. let suite = @@ -589,6 +871,34 @@ pub async fn user_cancel_task_suite( )))); } + // Cancel the suite *before* the archive of its tasks, not after. This statement + // locks the suite row. Each submit updates the same row with the condition + // `state <> Cancelled`. Thus, after this commit, a submit at the same time either + // came before the delete below, or the server rejects it. If the code writes the + // suite last, a submit between the delete and the write can add a task to a + // cancelled suite. No agent will claim that task, and `incomplete_tasks` is + // already zero. + // + // For the same reason, zero is the correct counter value. The delete below + // removes all active rows of the suite, and no request can add a new row. + let fenced = TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Cancelled), + ) + .col_expr(TaskSuites::Column::IncompleteTasks, Expr::value(0)) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .col_expr(TaskSuites::Column::CompletedAt, Expr::value(Some(now))) + .filter(TaskSuites::Column::Id.eq(suite.id)) + .filter(TaskSuites::Column::State.ne(TaskSuiteState::Cancelled)) + .exec(txn) + .await?; + if fenced.rows_affected == 0 { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Suite {suite_uuid} is already in cancelled state" + )))); + } + // All cancelled tasks share this result. let result = serde_json::to_value(TaskResultSpec { exit_status: 0, @@ -596,73 +906,123 @@ pub async fn user_cancel_task_suite( }) .inspect_err(|e| tracing::error!("{}", e))?; - // Archive every non-terminal task of the suite as Cancelled: - let tasks = ActiveTasks::Entity::delete_many() - .filter(ActiveTasks::Column::TaskSuiteId.eq(suite.id)) - .filter( - ActiveTasks::Column::State - .is_not_in([TaskState::Cancelled, TaskState::Unknown]), + // Do not return the rows to Rust. With many tasks, the rows cause the PostgreSQL + // error for too many arguments in the archive statement. + // + // The delete also removes `Cancelled` rows, and this is intentional. This state + // means that an agent reported `Cancel`, but did not send the `Commit` that + // archives the task. If these rows stay, they stay in `active_tasks` permanently. + // The terminal job will reject the `Commit`, no other agent can claim a task that + // is not `Ready`, and no process recovers this state. + let delete_stmt = DeleteStatement::new() + .from_table(ActiveTasks::Entity) + .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).eq(suite.id)) + .and_where( + Expr::col(ActiveTasks::Column::State).is_not_in([TaskState::Unknown]), ) - .exec_with_returning(txn) - .await?; + .returning_all() + .to_owned(); + let deleted_cte = CommonTableExpression::new() + .query(delete_stmt) + .table_name(Alias::new("deleted")) + .to_owned(); - // The inflight subset is the only set tied to an executing agent: each - // carries the `runner_uuid` of the agent running it. A Force cancel must - // signal those agents to stop so they don't commit tasks we just archived. - // Collected now and intentionally left unused. - // TODO: wire this to a per-agent shutdown/TasksCancelled - // push once the agent/notification layer exists - let _inflight_signals: Vec<(Option, Uuid)> = tasks - .iter() - .filter(|t| matches!(t.state, TaskState::Running | TaskState::Finished)) - .map(|t| (t.runner_uuid, t.uuid)) - .collect(); - - let cancelled_count = tasks.len() as u64; - if !tasks.is_empty() { - let archived: Vec = tasks - .into_iter() - .map(|task| ArchivedTasks::ActiveModel { - id: Set(task.id), - creator_id: Set(task.creator_id), - group_id: Set(task.group_id), - task_id: Set(task.task_id), - uuid: Set(task.uuid), - tags: Set(task.tags), - labels: Set(task.labels), - created_at: Set(task.created_at), - updated_at: Set(now), - state: Set(TaskState::Cancelled), - runner_uuid: Set(task.runner_uuid), - priority: Set(task.priority), - spec: Set(task.spec), - exec_options: Set(task.exec_options), - result: Set(Some(result.clone())), - upstream_task_uuid: Set(task.upstream_task_uuid), - downstream_task_uuid: Set(task.downstream_task_uuid), - task_suite_id: Set(task.task_suite_id), - }) - .collect(); - - ArchivedTasks::Entity::insert_many(archived) - .exec(txn) - .await?; - } + // The archived copy. It reads each column from `deleted` by name, and replaces + // `updated_at`, `state` and `result` with three values. The statement binds each of + // these values one time. + let select_from_deleted = Query::select() + .expr(Expr::col(Alias::new("id"))) + .expr(Expr::col(Alias::new("creator_id"))) + .expr(Expr::col(Alias::new("group_id"))) + .expr(Expr::col(Alias::new("task_id"))) + .expr(Expr::col(Alias::new("uuid"))) + .expr(Expr::col(Alias::new("tags"))) + .expr(Expr::col(Alias::new("labels"))) + .expr(Expr::col(Alias::new("created_at"))) + .expr(Expr::value(now)) + .expr(Expr::value(TaskState::Cancelled)) + .expr(Expr::col(Alias::new("runner_uuid"))) + .expr(Expr::col(Alias::new("exec_options"))) + .expr(Expr::col(Alias::new("priority"))) + .expr(Expr::col(Alias::new("spec"))) + .expr(Expr::value(result)) + .expr(Expr::col(Alias::new("upstream_task_uuid"))) + .expr(Expr::col(Alias::new("downstream_task_uuid"))) + .expr(Expr::col(Alias::new("task_suite_id"))) + .from(Alias::new("deleted")) + .to_owned(); + + let mut insert_stmt = InsertStatement::new() + .into_table(ArchivedTasks::Entity) + .columns([ + ArchivedTasks::Column::Id, + ArchivedTasks::Column::CreatorId, + ArchivedTasks::Column::GroupId, + ArchivedTasks::Column::TaskId, + ArchivedTasks::Column::Uuid, + ArchivedTasks::Column::Tags, + ArchivedTasks::Column::Labels, + ArchivedTasks::Column::CreatedAt, + ArchivedTasks::Column::UpdatedAt, + ArchivedTasks::Column::State, + ArchivedTasks::Column::RunnerUuid, + ArchivedTasks::Column::ExecOptions, + ArchivedTasks::Column::Priority, + ArchivedTasks::Column::Spec, + ArchivedTasks::Column::Result, + ArchivedTasks::Column::UpstreamTaskUuid, + ArchivedTasks::Column::DownstreamTaskUuid, + ArchivedTasks::Column::TaskSuiteId, + ]) + .to_owned(); + // This cannot fail: the column list and the select have the same number of + // columns. This is a property of the code, not of the request. + insert_stmt.select_from(select_from_deleted).unwrap(); + let archived_cte = CommonTableExpression::new() + .query(insert_stmt) + .table_name(Alias::new("archived")) + .to_owned(); + + // Agents hold only the tasks in this subset. The server tells these agents the + // UUIDs of the removed tasks. The agents then stop, and do not report on rows + // that this statement archived. + // + // This reads `deleted`, not `archived`, because it needs the state of each task + // *before* the archive changed it to `Cancelled`. The query does not use + // `archived`, and it does not have to: PostgreSQL runs a data-modifying CTE one + // time and completely, also when the main query does not read its output. + let in_flight_stmt = Query::select() + .expr(Expr::col(Alias::new("uuid"))) + .from(Alias::new("deleted")) + .and_where(Expr::col(Alias::new("state")).is_in([ + TaskState::Running, + TaskState::Finished, + TaskState::Cancelled, + ])) + .to_owned(); - let mut suite: TaskSuites::ActiveModel = suite.into(); - suite.state = Set(TaskSuiteState::Cancelled); - suite.updated_at = Set(now); - suite.completed_at = Set(Some(now)); - suite.update(txn).await?; + let with_ctes = WithClause::new() + .cte(deleted_cte) + .cte(archived_cte) + .to_owned(); + let cancelled_task_uuids: Vec = TaskUuidResult::find_by_statement( + builder.build(&in_flight_stmt.with(with_ctes)), + ) + .all(txn) + .await? + .into_iter() + .map(|t| t.uuid) + .collect(); - Ok(cancelled_count) + let suite_id = suite.id; + Ok((suite_id, cancelled_task_uuids)) }) }) .await?; - // TODO: notify assigned/executing agents that this suite was cancelled (and, for - // Force, push per-agent TasksCancelled built from `_running_signals` above), then - // drop the suite's in-memory task buffer in the dispatcher. + // TODO: when agents run suites, stop the jobs of `_suite_id`. `Force` kills the jobs, and + // `Graceful` lets them finish their current work. Send `SuiteCancelled` to each agent, and + // send `TasksCancelled` for `_cancelled_task_uuids`. Ok(()) } diff --git a/netmito/src/service/task.rs b/netmito/src/service/task.rs index e3a806b..243157a 100644 --- a/netmito/src/service/task.rs +++ b/netmito/src/service/task.rs @@ -1,10 +1,10 @@ use sea_orm::sea_query::{ extension::postgres::PgExpr, Alias, CommonTableExpression, DeleteStatement, ExprTrait, - InsertStatement, PgFunc, Query, + InsertStatement, LockType, Order, PgFunc, Query, WithClause, }; use sea_orm::ActiveValue::NotSet; -use sea_orm::{prelude::*, FromQueryResult, Set, TransactionTrait}; -use std::collections::HashSet; +use sea_orm::{prelude::*, ConnectionTrait, FromQueryResult, Set, TransactionTrait}; +use std::collections::{HashMap, HashSet}; use uuid::Uuid; use crate::entity::role::{GroupWorkerRole, UserGroupRole}; @@ -31,7 +31,7 @@ use super::worker::{remove_task, TaskDispatcherOp}; // XXX: Not sure if we can relax the constrains on local path checking. // We currently only check if the path is absolute or contains `..` and not check for `.`. -fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { +pub(crate) fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { if spec.resources.iter().any(|r| { r.local_path.is_absolute() || r.local_path @@ -45,9 +45,15 @@ fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { Ok(()) } +#[derive(Clone, Copy)] enum Submitter { User, - Worker(Uuid), + /// A running task that submits a downstream task. It has the UUID and the group of the parent + /// task. + Task { + upstream_task_uuid: Uuid, + parent_group_id: i64, + }, } async fn internal_submit_task( @@ -71,7 +77,9 @@ async fn internal_submit_task( // Used later when creating active tasks active model let (state, upstream_task_uuid) = match submitter { Submitter::User => (Set(crate::entity::state::TaskState::Ready), NotSet), - Submitter::Worker(upstream_task_uuid) => ( + Submitter::Task { + upstream_task_uuid, .. + } => ( Set(crate::entity::state::TaskState::Pending), Set(Some(upstream_task_uuid)), ), @@ -89,54 +97,19 @@ async fn internal_submit_task( .transaction::<_, (ActiveTasks::Model, Option), crate::error::Error>( |txn| { Box::pin(async move { - // Determine the owning group and (optionally) the suite, then - // atomically bump the group's task counter. When a suite is designated it - // is authoritative: the task's group is the suite's owning group, and the - // request's group_name must match it. - let (group_id, task_id, suite) = match suite_uuid { - None => { - // No suite: resolve the user-specified group, verify the caller's - // membership, and bump its task counter in one statement. - // No row back means either the group doesn't exist or the - // caller isn't in it; both return 404 so users won't know - // about the existence of a group they are not a member of. - // Membership must stay a subquery, not a join: both table has a - // `id` column and sea-orm will create a unqualified `RETURNING id`, - // which Postgres rejects as ambiguous. - let member_of = Query::select() - .column(UserGroup::Column::GroupId) - .from(UserGroup::Entity) - .and_where(Expr::col(UserGroup::Column::UserId).eq(creator_id)) - // TODO: when there is 'exec' access level, enforce access check here. - // - // .and_where( - // Expr::col(UserGroup::Column::Role) - // .gte(UserGroupRole::Write), - // ) - .to_owned(); - let group = Group::Entity::update_many() - .col_expr( - Group::Column::TaskCount, - Expr::col((Group::Entity, Group::Column::TaskCount)).add(1), - ) - .col_expr(Group::Column::UpdatedAt, Expr::value(now)) - .filter(Group::Column::GroupName.eq(&group_name)) - .filter(Group::Column::Id.in_subquery(member_of)) - .exec_with_returning(txn) - .await? - .into_iter() - .next() - .ok_or_else(|| { - Error::ApiError(crate::error::ApiError::NotFound(format!( - "User doesn't have permission or group with name {}", - group_name - ))) - })?; - (group.id, group.task_count, None) - } - Some(suite_uuid) => { - // Suite designated: resolve it (and the caller's membership in its - // owning group) in one join. The suite is authoritative for the group. + // Find the suite of the request, if there is one. The authorization depends on + // the submitter. A user can use a suite of a group that the user is a member + // of. A running task that submits a downstream task can use any suite of the + // group of the parent task. + // + // The server does not check the creator of the parent task again, and this is + // intentional. Thus, a suite that runs for a long time can continue to submit + // tasks after that user leaves the group. + let suite = match (suite_uuid, submitter) { + (None, _) => None, + (Some(suite_uuid), Submitter::User) => { + // Find the suite and the membership of the caller in the group of the + // suite with one join. let builder = txn.get_database_backend(); let suite_stmt = Query::select() .columns([ @@ -189,64 +162,194 @@ async fn internal_submit_task( suite_uuid ))) })?; - // The suite must be able to accept new tasks. - if !suite.state.can_accept_tasks() { - return Err(Error::ApiError( - crate::error::ApiError::InvalidRequest(format!( - "Suite is in {} state and cannot accept new tasks", - suite.state - )), - )); - } - // Derive the owning group from the suite and bump its counter. + Some(suite) + } + ( + Some(suite_uuid), + Submitter::Task { + parent_group_id, .. + }, + ) => Some( + TaskSuites::Entity::find() + .filter(TaskSuites::Column::Uuid.eq(suite_uuid)) + .filter(TaskSuites::Column::GroupId.eq(parent_group_id)) + .one(txn) + .await? + .ok_or_else(|| { + // For a suite of a different group, the error is the same as + // for a suite that does not exist. + Error::ApiError(crate::error::ApiError::NotFound(format!( + "Suite with uuid {} in the parent task's group", + suite_uuid + ))) + })?, + ), + }; + + // First, do a check without a lock. It rejects a suite that does not accept new + // tasks, and puts the actual state in the error message. The `WHERE` clause on + // the counter update below is the real check. This check only gives a better + // error message. + if let Some(suite) = &suite { + if !suite.state.can_accept_tasks() { + return Err(Error::ApiError(crate::error::ApiError::InvalidRequest( + format!( + "Suite is in {} state and cannot accept new tasks", + suite.state + ), + ))); + } + } + + // The group of the task, if a value other than `group_name` sets it. A suite in + // the request sets the group of the task, and a downstream task always stays in + // the group of its parent. `None` is the usual user path: the server finds the + // group from `group_name` and the membership of the caller. + let pinned_group_id = match (&suite, submitter) { + (Some(suite), _) => Some(suite.group_id), + ( + None, + Submitter::Task { + parent_group_id, .. + }, + ) => Some(parent_group_id), + (None, Submitter::User) => None, + }; + + // Increase the task counter of the group in one atomic statement. + let (group_id, task_id) = match pinned_group_id { + Some(pinned_group_id) => { let group = Group::Entity::update_many() .col_expr( Group::Column::TaskCount, Expr::col(Group::Column::TaskCount).add(1), ) .col_expr(Group::Column::UpdatedAt, Expr::value(now)) - .filter(Group::Column::Id.eq(suite.group_id)) + .filter(Group::Column::Id.eq(pinned_group_id)) .exec_with_returning(txn) .await? .into_iter() .next() .ok_or_else(|| { - // The suite's group row must exist (FK) - tracing::error!( - "Owning group {} of suite {} not found", - suite.group_id, - suite_uuid - ); + // The row must exist, because the suite and the parent task + // both have a foreign key to it. + tracing::error!("Owning group {} not found", pinned_group_id); Error::ApiError(crate::error::ApiError::InternalServerError) })?; - // The request's group_name must match the suite's owning group. - // Safe to reveal the real group: access to the suite is proven. + // The `group_name` of the request must be the same group. if group.group_name != group_name { + let owner = match &suite { + Some(suite) => { + format!( + "Suite {} belongs to group {}", + suite.uuid, group.group_name + ) + } + None => format!( + "The parent task belongs to group {}", + group.group_name + ), + }; return Err(Error::ApiError( crate::error::ApiError::InvalidRequest(format!( - "Suite {} belongs to group {}, not {}", - suite_uuid, group.group_name, group_name + "{}, not {}", + owner, group_name )), )); } - (group.id, group.task_count, Some(suite)) + (group.id, group.task_count) + } + None => { + // Find the group of the request, make sure that the caller is a member, + // and increase the task counter of the group, all in one statement. If + // no row comes back, the group does not exist or the caller is not a + // member. The two cases return 404, so that a user cannot find out if a + // group exists when the user is not a member. + // The membership check must be a subquery, not a join. The two tables + // have an `id` column, and sea-orm makes an unqualified `RETURNING id`. + // PostgreSQL rejects it as ambiguous. + let member_of = Query::select() + .column(UserGroup::Column::GroupId) + .from(UserGroup::Entity) + .and_where(Expr::col(UserGroup::Column::UserId).eq(creator_id)) + // TODO: when there is an 'exec' access level, do the access check + // here. + // + // .and_where( + // Expr::col(UserGroup::Column::Role) + // .gte(UserGroupRole::Write), + // ) + .to_owned(); + let group = Group::Entity::update_many() + .col_expr( + Group::Column::TaskCount, + Expr::col((Group::Entity, Group::Column::TaskCount)).add(1), + ) + .col_expr(Group::Column::UpdatedAt, Expr::value(now)) + .filter(Group::Column::GroupName.eq(&group_name)) + .filter(Group::Column::Id.in_subquery(member_of)) + .exec_with_returning(txn) + .await? + .into_iter() + .next() + .ok_or_else(|| { + Error::ApiError(crate::error::ApiError::NotFound(format!( + "User doesn't have permission or group with name {}", + group_name + ))) + })?; + (group.id, group.task_count) } }; + // Use one statement, so that the database calculates the two increases on the + // row that this `UPDATE` locks. If the code reads the counters first and writes + // the values back, two submits at the same time both read `total = n` and both + // write `n + 1`. Then the counter of the suite is too low, and + // `decrement_incomplete_tasks` gets to zero while tasks are still `Ready`. + // These tasks then stay in a `Complete` suite that gives out no tasks. + // + // The condition on the state does the check that the suite accepts tasks. Thus, + // the statement does not update the row if a cancel committed after the read + // above. let suite = match suite { None => None, - Some(suite) => { - let prev_total_tasks = suite.total_tasks; - let prev_incomplete_tasks = suite.incomplete_tasks; - let mut suite: TaskSuites::ActiveModel = suite.into(); - suite.total_tasks = Set(prev_total_tasks + 1); - suite.incomplete_tasks = Set(prev_incomplete_tasks + 1); - suite.last_task_submitted_at = Set(Some(now)); - suite.updated_at = Set(now); - suite.state = Set(TaskSuiteState::Open); - suite.completed_at = Set(None); - Some(suite.update(txn).await?) - } + Some(suite) => Some( + TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::TotalTasks, + Expr::col(TaskSuites::Column::TotalTasks).add(1), + ) + .col_expr( + TaskSuites::Column::IncompleteTasks, + Expr::col(TaskSuites::Column::IncompleteTasks).add(1), + ) + .col_expr( + TaskSuites::Column::LastTaskSubmittedAt, + Expr::value(Some(now)), + ) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Open), + ) + .col_expr( + TaskSuites::Column::CompletedAt, + Expr::value(None::), + ) + .filter(TaskSuites::Column::Id.eq(suite.id)) + .filter(TaskSuites::Column::State.ne(TaskSuiteState::Cancelled)) + .exec_with_returning(txn) + .await? + .into_iter() + .next() + .ok_or_else(|| { + // A cancel committed after the read above. + Error::ApiError(crate::error::ApiError::InvalidRequest( + format!("Suite {} stopped accepting new tasks", suite.uuid), + )) + })?, + ), }; let task_uuid = Uuid::new_v4(); @@ -267,9 +370,23 @@ async fn internal_submit_task( exec_options: Set(exec_options_json), result: Set(None), upstream_task_uuid, + task_suite_id: Set(suite.as_ref().map(|s| s.id)), ..Default::default() }; let task = task.insert(txn).await?; + + // Push the task to the queue here, in the transaction, and not after the + // commit. `served` empties the queue: it removes the tasks that a claim locked. + // A claim cannot lock a row before the row is committed. Thus, a task that is + // pushed before its commit is always pushed before a `served` call that can + // remove it. If the push occurs after the commit, a claim can come between the + // commit and the push. The claim takes the task, and `served` finds nothing to + // remove. Then the push adds a running task to the backlog of the suite, and no + // process removes it. + // + // TODO: when suites have a ready queue, push a `Ready` suite task to it here + // with `queues.push_ready(suite.id, [(task.id, task.priority)])`. + Ok((task, suite)) }) }, @@ -339,13 +456,28 @@ pub async fn user_submit_task( internal_submit_task(pool, creator_id, Submitter::User, req).await } +/// Submit a downstream task of a running task, for the worker or agent that runs the parent task. +/// +/// The downstream task stays in the group of its parent. `req.suite_uuid` can be any suite of that +/// group: the suite of the parent, a different suite, or `None` for a task without a suite that +/// goes to workers. pub async fn worker_submit_pending_task( pool: &InfraPool, creator_id: i64, upstream_task_uuid: Uuid, + parent_group_id: i64, req: SubmitTaskReq, ) -> crate::error::Result { - internal_submit_task(pool, creator_id, Submitter::Worker(upstream_task_uuid), req).await + internal_submit_task( + pool, + creator_id, + Submitter::Task { + upstream_task_uuid, + parent_group_id, + }, + req, + ) + .await } pub async fn worker_trigger_pending_task(pool: &InfraPool, uuid: Uuid) -> crate::error::Result<()> { @@ -362,6 +494,13 @@ pub async fn worker_trigger_pending_task(pool: &InfraPool, uuid: Uuid) -> crate: task.state = Set(TaskState::Ready); task.updated_at = Set(TimeDateTimeWithTimeZone::now_utc()); let task = task.update(&pool.db).await?; + // A suite task does not go into the worker queues. Agents get it from its suite. Thus, wake the + // idle agents that can run the suite. + if let Some(_suite_id) = task.task_suite_id { + // TODO: when agents can get notifications, call `notify_suite_available(pool, _suite_id)` + // here. + return Ok(()); + } // Batch add task to worker task queues let builder = pool.db.get_database_backend(); let tasks_stmt = Query::select() @@ -613,6 +752,7 @@ pub async fn user_cancel_task( } } let now = TimeDateTimeWithTimeZone::now_utc(); + let suite_id = task.task_suite_id; let res = TaskResultSpec { exit_status: 0, msg: Some(crate::schema::TaskResultMessage::UserCancellation), @@ -638,6 +778,13 @@ pub async fn user_cancel_task( downstream_task_uuid: Set(task.downstream_task_uuid), task_suite_id: Set(task.task_suite_id), }; + // A cancelled suite task does not get to an agent `Commit`. Thus, this is the only + // point where the suite can get its count back. This step comes first, so that the + // suite row is locked before the task rows. All writers use this order. + if let Some(suite_id) = suite_id { + crate::service::suite::decrement_incomplete_tasks(txn, suite_id, 1, now) + .await?; + } archived_task.insert(txn).await?; ActiveTasks::Entity::delete_by_id(task.id).exec(txn).await?; Ok(task.id) @@ -646,6 +793,7 @@ pub async fn user_cancel_task( .await?; let _ = remove_task(task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); + // TODO: when suites have a ready queue, also remove a cancelled suite task from it. Ok(()) } @@ -675,6 +823,10 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), + ) .from(ActiveTasks::Entity) .join( sea_orm::JoinType::Join, @@ -690,6 +842,15 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((ActiveTasks::Entity, ActiveTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) + // Use a left join: most tasks do not have a suite, and the query must keep these rows. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ActiveTasks::Entity, + ActiveTasks::Column::TaskSuiteId, + ))), + ) .and_where(Expr::col((ActiveTasks::Entity, ActiveTasks::Column::Uuid)).eq(uuid)) .limit(1) .to_owned(); @@ -724,6 +885,10 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), + ) .from(ArchivedTasks::Entity) .join( sea_orm::JoinType::Join, @@ -739,6 +904,15 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) + // Use a left join: most tasks do not have a suite, and the query must keep these rows. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ArchivedTasks::Entity, + ArchivedTasks::Column::TaskSuiteId, + ))), + ) .and_where(Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::Uuid)).eq(uuid)) .limit(1) .to_owned(); @@ -768,6 +942,7 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res uuid: info.uuid, creator_username: info.creator_username, group_name: info.group_name, + suite_uuid: info.suite_uuid, task_id: info.task_id, tags: info.tags, labels: info.labels, @@ -790,12 +965,21 @@ struct UserGroupRoleQueryRes { role: UserGroupRole, } +/// The suite that limits a query, together with the group of the suite, so that the code can +/// compare the two. +#[derive(FromQueryResult)] +struct SuiteIdWithGroup { + id: i64, + group_name: String, +} + +/// Returns the id of the suite in the query, for [`apply_task_filters`]. pub(crate) async fn check_task_list_query( user_id: i64, pool: &InfraPool, query: &mut TasksQueryReq, role: UserGroupRole, -) -> crate::error::Result<()> { +) -> crate::error::Result> { if let Some(ref tags) = query.tags { if tags.is_empty() { return Err(Error::ApiError(crate::error::ApiError::InvalidRequest( @@ -824,6 +1008,69 @@ pub(crate) async fn check_task_list_query( ))); } } + // A suite is in exactly one group, and a task submitted to the suite gets that group. Thus, the + // suite identifies the group, and does not make the group filter smaller. If the caller gives + // no group, the server uses the group of the suite. If the caller gives a different group, the + // query fails. The server finds the suite through the memberships of the caller. Thus, the + // answer is the same for a suite in a group that the caller is not in, a suite that does not + // exist, and a suite that is not in `group_name`. The answer does not show the group of the + // suite, because a UUID alone must not show it. + let suite_id = match query.suite_uuid { + None => None, + Some(suite_uuid) => { + let builder = pool.db.get_database_backend(); + let suite_stmt = Query::select() + .column((TaskSuites::Entity, TaskSuites::Column::Id)) + .expr_as( + Expr::col((Group::Entity, Group::Column::GroupName)), + Alias::new("group_name"), + ) + .from(TaskSuites::Entity) + .join( + sea_orm::JoinType::Join, + Group::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::GroupId)) + .eq(Expr::col((Group::Entity, Group::Column::Id))), + ) + .join( + sea_orm::JoinType::Join, + UserGroup::Entity, + Expr::col((UserGroup::Entity, UserGroup::Column::GroupId)) + .eq(Expr::col((TaskSuites::Entity, TaskSuites::Column::GroupId))), + ) + .and_where(Expr::col((UserGroup::Entity, UserGroup::Column::UserId)).eq(user_id)) + .and_where(Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)).eq(suite_uuid)) + .to_owned(); + let suite = SuiteIdWithGroup::find_by_statement(builder.build(&suite_stmt)) + .one(&pool.db) + .await? + .ok_or_else(|| { + // This occurs when the query has no result: the suite does not exist, or the + // user cannot access it. + Error::ApiError(crate::error::ApiError::NotFound(format!( + "User doesn't have permission or suite with uuid {suite_uuid}" + ))) + })?; + match query.group_name { + Some(ref group_name) if *group_name != suite.group_name => { + // This occurs when the `group_name` of the request is not the group of the + // suite. + return Err(Error::ApiError(crate::error::ApiError::NotFound(format!( + "User doesn't have permission or suite with uuid {suite_uuid}" + )))); + } + Some(_) => {} + None => { + tracing::debug!( + "No group name specified, use the suite's group {} instead", + suite.group_name + ); + query.group_name = Some(suite.group_name); + } + } + Some(suite.id) + } + }; if query.group_name.is_none() { let username = User::Entity::find() .filter(User::Column::Id.eq(user_id)) @@ -866,7 +1113,7 @@ pub(crate) async fn check_task_list_query( } } } - Ok(()) + Ok(suite_id) } #[derive(Debug, Copy, Clone)] @@ -905,7 +1152,19 @@ pub(crate) fn apply_task_filters( active_stmt: &mut sea_orm::sea_query::SelectStatement, archive_stmt: &mut sea_orm::sea_query::SelectStatement, query: &TasksQueryReq, + suite_id: Option, ) -> crate::error::Result<()> { + // Use the id, not `query.suite_uuid`. `check_task_list_query` read the row for the + // authorization, and the archive has the same column. Thus, the two statements do not need a + // join on the suite table. + if let Some(suite_id) = suite_id { + active_stmt.and_where( + Expr::col((ActiveTasks::Entity, ActiveTasks::Column::TaskSuiteId)).eq(suite_id), + ); + archive_stmt.and_where( + Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::TaskSuiteId)).eq(suite_id), + ); + } if let Some(runner_uuid) = query.runner_uuid { active_stmt.and_where( Expr::col((ActiveTasks::Entity, ActiveTasks::Column::RunnerUuid)).eq(runner_uuid), @@ -1105,7 +1364,7 @@ pub async fn query_tasks_by_filter( pool: &InfraPool, mut query: TasksQueryReq, ) -> crate::error::Result { - check_task_list_query(user_id, pool, &mut query, UserGroupRole::Read).await?; + let suite_id = check_task_list_query(user_id, pool, &mut query, UserGroupRole::Read).await?; let mut active_stmt = Query::select(); if query.count { active_stmt.expr(Expr::col((ActiveTasks::Entity, ActiveTasks::Column::Uuid)).count()); @@ -1134,6 +1393,10 @@ pub async fn query_tasks_by_filter( .expr_as( Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), + ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), ); } active_stmt @@ -1151,6 +1414,15 @@ pub async fn query_tasks_by_filter( Group::Entity, Expr::col((ActiveTasks::Entity, ActiveTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), + ) + // Use a left join: most tasks do not have a suite, and the query must keep these rows. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ActiveTasks::Entity, + ActiveTasks::Column::TaskSuiteId, + ))), ); let mut archive_stmt = Query::select(); if query.count { @@ -1186,6 +1458,10 @@ pub async fn query_tasks_by_filter( .expr_as( Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), + ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), ); } archive_stmt @@ -1203,10 +1479,19 @@ pub async fn query_tasks_by_filter( Group::Entity, Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), + ) + // Use a left join: most tasks do not have a suite, and the query must keep these rows. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ArchivedTasks::Entity, + ArchivedTasks::Column::TaskSuiteId, + ))), ); // Apply filters using the shared helper function - apply_task_filters(&mut active_stmt, &mut archive_stmt, &query)?; + apply_task_filters(&mut active_stmt, &mut archive_stmt, &query, suite_id)?; let builder = pool.db.get_database_backend(); let resp = if query.count { let active_count = CountQuery::find_by_statement(builder.build(&active_stmt)) @@ -1244,6 +1529,33 @@ pub async fn query_tasks_by_filter( #[derive(FromQueryResult)] struct IdResult { id: i64, + task_suite_id: Option, +} + +/// Decrease `incomplete_tasks` by one for each cancelled task, for each suite that a batch cancel +/// changed. +/// +/// The rows come directly from the `RETURNING` of the archive. Thus, the count is the number of +/// tasks that the cancel really removed, not the number in the request. +async fn decrement_incomplete_tasks_by_suite( + txn: &C, + suite_ids: impl IntoIterator>, + now: TimeDateTimeWithTimeZone, +) -> crate::error::Result<()> { + let mut per_suite: HashMap = HashMap::new(); + for suite_id in suite_ids.into_iter().flatten() { + *per_suite.entry(suite_id).or_default() += 1; + } + // Sort the suites, and do not use the map order. Each decrease writes its suite row and keeps + // the lock until the commit. If two cancels change the same suites in opposite order, they + // deadlock. Usually the `locked` CTE of each batch cancel already locked these rows, so this + // sort is a second safety step. It is still necessary for a suite that the CTE cannot lock. + let mut per_suite: Vec<(i64, i32)> = per_suite.into_iter().collect(); + per_suite.sort_unstable_by_key(|(suite_id, _)| *suite_id); + for (suite_id, count) in per_suite { + crate::service::suite::decrement_incomplete_tasks(txn, suite_id, count, now).await?; + } + Ok(()) } /// Cancel multiple tasks by filter criteria. @@ -1258,6 +1570,7 @@ pub async fn cancel_tasks_by_filter( let mut query = TasksQueryReq { creator_usernames: req.creator_usernames, group_name: req.group_name, + suite_uuid: req.suite_uuid, tags: req.tags, labels: req.labels, runner_uuid: None, @@ -1288,7 +1601,7 @@ pub async fn cancel_tasks_by_filter( }; // Validate query and fill in defaults (also checks Write permission) - check_task_list_query(user_id, pool, &mut query, UserGroupRole::Write).await?; + let suite_id = check_task_list_query(user_id, pool, &mut query, UserGroupRole::Write).await?; let group_name = query.group_name.clone().unwrap_or_default(); // Build task ID subquery with the same filters (avoids parameter limits) @@ -1313,21 +1626,78 @@ pub async fn cancel_tasks_by_filter( // Apply the same filters to the subquery let mut dummy_archive_stmt = Query::select().from(ArchivedTasks::Entity).to_owned(); - apply_task_filters(&mut task_id_subquery, &mut dummy_archive_stmt, &query)?; + apply_task_filters( + &mut task_id_subquery, + &mut dummy_archive_stmt, + &query, + suite_id, + )?; // Build the complete CTE statement before the transaction to avoid lifetime issues + // The suite rows that this cancel will decrease. All other writers lock the suite row before + // the task rows, and a batch cancel uses this CTE to do the same. Without it, the cancel finds + // its suites only from the `RETURNING` of the archive, and that is too late to lock them. + // + // This CTE is in the same statement as the delete, so one snapshot applies to the two. The + // filter selects the same tasks in each CTE, and the CTEs do not see a task that is submitted + // between them. + // + // `ORDER BY id` is necessary. PostgreSQL puts `LockRows` above the sort, so it locks the rows + // in id order. Thus, each scan locks rows in ascending id order, and two cancels cannot lock + // two rows in opposite order. + let locked_suites = Query::select() + .column(TaskSuites::Column::Id) + .from(TaskSuites::Entity) + .and_where( + Expr::col(TaskSuites::Column::Id).in_subquery( + Query::select() + .column(ActiveTasks::Column::TaskSuiteId) + .from(ActiveTasks::Entity) + .and_where( + Expr::col(ActiveTasks::Column::Id).in_subquery(task_id_subquery.clone()), + ) + .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).is_not_null()) + .to_owned(), + ), + ) + .order_by(TaskSuites::Column::Id, Order::Asc) + .lock(LockType::Update) + .to_owned(); + let locked_cte = CommonTableExpression::new() + .query(locked_suites) + .table_name(Alias::new("locked")) + .to_owned(); + let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::Id).in_subquery(task_id_subquery)) + // This condition is the guarantee, not an extra check. The statement deletes a task only if + // its suite is in `locked`, and `locked` locks each row that it produces. Thus, the suite + // row is always locked before the task row, for all query plans of the subquery. + .and_where( + Expr::col(ActiveTasks::Column::TaskSuiteId) + .is_null() + .or(Expr::col(ActiveTasks::Column::TaskSuiteId).in_subquery( + Query::select() + .expr(Expr::col(Alias::new("id"))) + .from(Alias::new("locked")) + .to_owned(), + )), + ) .returning_all() .to_owned(); - let cte = CommonTableExpression::new() + let deleted_cte = CommonTableExpression::new() .query(delete_stmt) .table_name(Alias::new("deleted")) .to_owned(); + let with_ctes = WithClause::new() + .cte(locked_cte) + .cte(deleted_cte) + .to_owned(); + // Get the database backend before the transaction let builder = pool.db.get_database_backend(); @@ -1336,7 +1706,7 @@ pub async fn cancel_tasks_by_filter( let task_ids = pool .db .transaction::<_, Vec, crate::error::Error>(|txn| { - let cte = cte.clone(); + let with_ctes = with_ctes.clone(); Box::pin(async move { let now = TimeDateTimeWithTimeZone::now_utc(); let res = TaskResultSpec { @@ -1364,6 +1734,7 @@ pub async fn cancel_tasks_by_filter( .expr(Expr::value(result.clone())) .expr(Expr::col(Alias::new("upstream_task_uuid"))) .expr(Expr::col(Alias::new("downstream_task_uuid"))) + .expr(Expr::col(Alias::new("task_suite_id"))) .from(Alias::new("deleted")) .to_owned(); @@ -1388,21 +1759,28 @@ pub async fn cancel_tasks_by_filter( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, + // Without this column, the archived copy of a suite task does not have its + // suite, and the count below cannot group the tasks by suite. + ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); insert_stmt.select_from(select_from_cte).unwrap(); - insert_stmt.returning_col(ArchivedTasks::Column::Id); - let insert_with_cte = insert_stmt.with(cte.into()); + insert_stmt.returning_all(); + let insert_with_cte = insert_stmt.with(with_ctes); let stmt = builder.build(&insert_with_cte); - let task_ids: Vec = IdResult::find_by_statement(stmt) - .all(txn) - .await? - .into_iter() - .map(|r| r.id) - .collect(); + let cancelled: Vec = IdResult::find_by_statement(stmt).all(txn).await?; + + decrement_incomplete_tasks_by_suite( + txn, + cancelled.iter().map(|r| r.task_suite_id), + now, + ) + .await?; + + let task_ids: Vec = cancelled.into_iter().map(|r| r.id).collect(); Ok(task_ids) }) @@ -1414,6 +1792,7 @@ pub async fn cancel_tasks_by_filter( let _ = remove_task(*task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); } + // TODO: when suites have a ready queue, also remove the cancelled suite tasks from it. Ok(TasksCancelByFilterResp { cancelled_count: task_ids.len() as u64, @@ -1425,6 +1804,7 @@ pub async fn cancel_tasks_by_filter( struct IdUuidResult { id: i64, uuid: Uuid, + task_suite_id: Option, } pub async fn cancel_tasks_by_uuids( @@ -1488,23 +1868,77 @@ pub async fn cancel_tasks_by_uuids( // Build CTE for DELETE RETURNING + INSERT SELECT to avoid parameter limits // Convert the SELECT statement into a subquery for the DELETE + // The suite rows that this cancel will decrease. All other writers lock the suite row + // before the task rows, and a batch cancel uses this CTE to do the same. Without it, the + // cancel finds its suites only from the `RETURNING` of the archive, and that is too late to + // lock them. + // + // This CTE is in the same statement as the delete, so one snapshot applies to the two. The + // filter selects the same tasks in each CTE, and the CTEs do not see a task that is + // submitted between them. + // + // `ORDER BY id` is necessary. PostgreSQL puts `LockRows` above the sort, so it locks the + // rows in id order. Thus, each scan locks rows in ascending id order, and two cancels + // cannot lock two rows in opposite order. + let locked_suites = Query::select() + .column(TaskSuites::Column::Id) + .from(TaskSuites::Entity) + .and_where( + Expr::col(TaskSuites::Column::Id).in_subquery( + Query::select() + .column(ActiveTasks::Column::TaskSuiteId) + .from(ActiveTasks::Entity) + .and_where( + Expr::col(ActiveTasks::Column::Id).in_subquery(id_subquery.clone()), + ) + .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).is_not_null()) + .to_owned(), + ), + ) + .order_by(TaskSuites::Column::Id, Order::Asc) + .lock(LockType::Update) + .to_owned(); + let locked_cte = CommonTableExpression::new() + .query(locked_suites) + .table_name(Alias::new("locked")) + .to_owned(); + let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::Id).in_subquery(id_subquery)) + // This condition is the guarantee, not an extra check. The statement deletes a task + // only if its suite is in `locked`, and `locked` locks each row that it produces. Thus, + // the suite row is always locked before the task row, for all query plans of the + // subquery. + .and_where( + Expr::col(ActiveTasks::Column::TaskSuiteId) + .is_null() + .or(Expr::col(ActiveTasks::Column::TaskSuiteId).in_subquery( + Query::select() + .expr(Expr::col(Alias::new("id"))) + .from(Alias::new("locked")) + .to_owned(), + )), + ) .returning_all() .to_owned(); - let cte = CommonTableExpression::new() + let deleted_cte = CommonTableExpression::new() .query(delete_stmt) .table_name(Alias::new("deleted")) .to_owned(); + let with_ctes = WithClause::new() + .cte(locked_cte) + .cte(deleted_cte) + .to_owned(); + // Execute delete and insert in a single transaction // Total: 1 query using CTE (DELETE RETURNING + INSERT SELECT) per chunk let (task_ids, found_uuids) = pool .db .transaction::<_, (Vec, HashSet), crate::error::Error>(|txn| { - let cte = cte.clone(); + let with_ctes = with_ctes.clone(); Box::pin(async move { let now = TimeDateTimeWithTimeZone::now_utc(); let res = TaskResultSpec { @@ -1533,6 +1967,7 @@ pub async fn cancel_tasks_by_uuids( .expr(Expr::value(result.clone())) .expr(Expr::col(Alias::new("upstream_task_uuid"))) .expr(Expr::col(Alias::new("downstream_task_uuid"))) + .expr(Expr::col(Alias::new("task_suite_id"))) .from(Alias::new("deleted")) .to_owned(); @@ -1557,18 +1992,28 @@ pub async fn cancel_tasks_by_uuids( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, + // Without this column, the archived copy of a suite task does not have + // its suite, and the count below cannot group the tasks by suite. + ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); insert_stmt.select_from(select_from_cte).unwrap(); insert_stmt.returning_all(); - let insert_with_cte = insert_stmt.with(cte.into()); + let insert_with_cte = insert_stmt.with(with_ctes); let stmt = builder.build(&insert_with_cte); let results: Vec = IdUuidResult::find_by_statement(stmt).all(txn).await?; + decrement_incomplete_tasks_by_suite( + txn, + results.iter().map(|r| r.task_suite_id), + now, + ) + .await?; + let task_ids: Vec = results.iter().map(|r| r.id).collect(); let found_uuids: HashSet = results.into_iter().map(|r| r.uuid).collect(); @@ -1587,6 +2032,7 @@ pub async fn cancel_tasks_by_uuids( let _ = remove_task(*task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); } + // TODO: when suites have a ready queue, also remove the cancelled suite tasks from it. // Determine which UUIDs failed (not found or no permission) let failed_uuids: Vec = req diff --git a/netmito/src/service/worker/mod.rs b/netmito/src/service/worker/mod.rs index 48298aa..6660b0b 100644 --- a/netmito/src/service/worker/mod.rs +++ b/netmito/src/service/worker/mod.rs @@ -642,19 +642,8 @@ pub async fn report_task( if user.state != UserState::Active { return Err(AuthError::PermissionDenied.into()); } - // Load group information to verify group name matches - let group = Group::Entity::find_by_id(task.group_id) - .one(&pool.db) - .await? - .ok_or(ApiError::NotFound("Group not found".to_string()))?; - if req.group_name != group.group_name { - return Err(ApiError::InvalidRequest(format!( - "Group name mismatch: expected '{}', got '{}'", - group.group_name, req.group_name - )) - .into()); - } - // Verify worker has permission to access this group + // Make sure that the worker can access this group. The submit path gives the + // downstream task the group of its parent, and rejects a different `req.group_name`. GroupWorker::Entity::find() .filter(GroupWorker::Column::WorkerId.eq(worker_id)) .filter(GroupWorker::Column::GroupId.eq(task.group_id)) @@ -664,17 +653,22 @@ pub async fn report_task( .ok_or(ApiError::AuthError( crate::error::AuthError::PermissionDenied, ))?; - let resp = - service::task::worker_submit_pending_task(pool, task.creator_id, task.uuid, req) - .await - .map_err(|e| match e { - crate::error::Error::AuthError(err) => ApiError::AuthError(err), - crate::error::Error::ApiError(e) => e, - _ => { - tracing::error!("{}", e); - ApiError::InternalServerError - } - })?; + let resp = service::task::worker_submit_pending_task( + pool, + task.creator_id, + task.uuid, + task.group_id, + req, + ) + .await + .map_err(|e| match e { + crate::error::Error::AuthError(err) => ApiError::AuthError(err), + crate::error::Error::ApiError(e) => e, + _ => { + tracing::error!("{}", e); + ApiError::InternalServerError + } + })?; let task = ActiveTask::ActiveModel { id: Set(task_id), updated_at: Set(now), @@ -718,10 +712,20 @@ pub async fn report_task( assigned_task_id: Set(None), ..Default::default() }; + let suite_id = task.task_suite_id; + // Decrease the suite counter in the same transaction that archives the task. + // With two transactions, a failure between them keeps `incomplete_tasks` too + // high permanently. Then the counter never gets to zero, and the suite never + // completes. Decrease the counter first. This locks the suite row before the + // task rows, which is the lock order that all writers use. let (worker_id, worker_state) = pool .db - .transaction(|txn| { + .transaction::<_, (i64, WorkerState), Error>(|txn| { Box::pin(async move { + if let Some(suite_id) = suite_id { + service::suite::decrement_incomplete_tasks(txn, suite_id, 1, now) + .await?; + } archived_task.insert(txn).await?; ActiveTask::Entity::delete_by_id(task_id).exec(txn).await?; let worker = worker.update(txn).await?; diff --git a/openapi.yaml b/openapi.yaml index f73ba3e..ded4a20 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1644,6 +1644,47 @@ paths: "404": $ref: "#/components/responses/NotFound" + put: + summary: Change task suite + description: >- + Change the properties of a suite. All fields are optional, and an absent field + does not change. The server rejects an empty request. A nullable field (name, + description and each hook) takes an update op, not a plain value, so that you can + clear it. You cannot change a Cancelled suite, and this includes its name. The + change applies to the jobs that start after it, because a job copies the worker + schedule and the hooks when it starts. The caller must have the Write or Admin + role in the group of the suite. If the caller is not a member, or the UUID does + not exist, the server returns 404 for the two cases. + operationId: changeTaskSuite + tags: + - Suites + security: + - bearerAuth: [] + parameters: + - name: uuid + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ChangeTaskSuiteReq" + responses: + "200": + description: Suite changed successfully + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + delete: summary: Cancel task suite description: Cancel a task suite (terminal). Archives all non-terminal tasks of the suite as Cancelled. @@ -3278,6 +3319,17 @@ components: nullable: true description: Filter by group name example: research-team + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -3359,6 +3411,11 @@ components: group_name: type: string example: research-team + suite_uuid: + type: string + format: uuid + nullable: true + description: The task suite of this task, if the task is in a suite task_id: type: integer format: int64 @@ -3439,6 +3496,11 @@ components: type: string group_name: type: string + suite_uuid: + type: string + format: uuid + nullable: true + description: The task suite of this task, if the task is in a suite task_id: type: integer format: int64 @@ -3677,6 +3739,17 @@ components: type: string nullable: true description: Filter by group name + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -3867,6 +3940,17 @@ components: type: string nullable: true description: Filter by group name + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -4045,6 +4129,17 @@ components: type: string nullable: true description: Filter by group name + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -4370,6 +4465,94 @@ components: exec_hooks: $ref: "#/components/schemas/ExecHooks" + UpdateOpString: + type: object + description: >- + A change to a nullable string field. `set_to` writes the value, and `set_null` + clears it. If the request does not contain the field, the field does not change. + required: + - op + properties: + op: + type: string + enum: + - set_to + - set_null + value: + type: string + description: Necessary when op is set_to. Absent for the other ops + example: + op: set_to + value: nightly-eval + + UpdateOpExecSpec: + type: object + description: >- + A change to one lifecycle hook. `set_to` sets the spec, and `set_null` removes the + hook. If the request does not contain the field, the hook does not change. + required: + - op + properties: + op: + type: string + enum: + - set_to + - set_null + value: + type: object + description: The execution spec. Necessary when op is set_to + + ExecHooksUpdate: + type: object + description: >- + The changes to the hooks, one for each hook. You can set a hook, clear a hook, or + leave a hook out to keep it. If you clear all three hooks, the suite stores no + hooks. + properties: + provision: + $ref: "#/components/schemas/UpdateOpExecSpec" + cleanup: + $ref: "#/components/schemas/UpdateOpExecSpec" + background: + $ref: "#/components/schemas/UpdateOpExecSpec" + + ChangeTaskSuiteReq: + type: object + description: At least one field must be present. The server rejects an empty request + properties: + name: + $ref: "#/components/schemas/UpdateOpString" + description: + $ref: "#/components/schemas/UpdateOpString" + tags: + type: array + items: + type: string + description: >- + The new tags that an agent must have to run the suite. An empty array matches + all agents that the group of the suite can write to, because the match is a + containment check + example: + - linux + - cuda:11 + labels: + type: array + items: + type: string + description: The new labels, which you can use to query and filter suites + priority: + type: integer + format: int32 + description: Suite scheduling priority (higher = more important) + worker_schedule: + allOf: + - $ref: "#/components/schemas/WorkerSchedulePlan" + description: >- + Replaces the full plan. A partial change cannot change the plan to a different + scheduling policy + exec_hooks: + $ref: "#/components/schemas/ExecHooksUpdate" + CreateTaskSuiteResp: type: object required: