From 736ba046314c6fea1c6d39e76718708d9cb0a57a Mon Sep 17 00:00:00 2001 From: un-lock-able Date: Fri, 16 Oct 2026 16:44:16 +0800 Subject: [PATCH 1/4] feat(task): filter tasks and artifacts by suite uuid --- netmito/src/client/interactive.rs | 6 + netmito/src/client/mod.rs | 15 ++- netmito/src/config/client/artifacts.rs | 10 ++ netmito/src/config/client/tasks.rs | 10 ++ netmito/src/schema/artifact.rs | 2 + netmito/src/schema/task.rs | 9 ++ netmito/src/service/s3.rs | 2 + netmito/src/service/task.rs | 159 ++++++++++++++++++++++++- openapi.yaml | 50 ++++++++ 9 files changed, 256 insertions(+), 7 deletions(-) diff --git a/netmito/src/client/interactive.rs b/netmito/src/client/interactive.rs index d3e8898..c017a8b 100644 --- a/netmito/src/client/interactive.rs +++ b/netmito/src/client/interactive.rs @@ -111,6 +111,9 @@ pub(crate) fn output_parsed_task_info(info: &ParsedTaskQueryInfo) { if let Some(downstream_task_uuid) = info.downstream_task_uuid { tracing::info!("Downstream Task UUID: {:?}", downstream_task_uuid); } + if let Some(suite_uuid) = info.suite_uuid { + tracing::info!("Task Suite UUID: {}", suite_uuid); + } if let Some(runner_uuid) = info.runner_uuid { tracing::info!("Runner UUID: {}", runner_uuid); } @@ -149,6 +152,9 @@ pub(crate) fn output_task_info(info: &TaskQueryInfo) { if let Some(downstream_task_uuid) = info.downstream_task_uuid { tracing::info!("Downstream Task UUID: {:?}", downstream_task_uuid); } + if let Some(suite_uuid) = info.suite_uuid { + tracing::info!("Task Suite UUID: {}", suite_uuid); + } if let Some(runner_uuid) = info.runner_uuid { tracing::info!("Runner UUID: {}", runner_uuid); } else { diff --git a/netmito/src/client/mod.rs b/netmito/src/client/mod.rs index b818005..097ac20 100644 --- a/netmito/src/client/mod.rs +++ b/netmito/src/client/mod.rs @@ -1879,8 +1879,21 @@ impl MitoClient { output_task_info(&task); } } else { + // Bare uuids, except that a suite task says + // so: without it the terse listing is the + // one place a suite task is + // indistinguishable from a loose one. for task in resp.tasks { - tracing::info!("{}", task.uuid); + match task.suite_uuid { + Some(suite_uuid) => { + tracing::info!( + "{}, suite {}", + task.uuid, + suite_uuid + ) + } + None => tracing::info!("{}", task.uuid), + } } } } diff --git a/netmito/src/config/client/artifacts.rs b/netmito/src/config/client/artifacts.rs index 5766d08..be5db64 100644 --- a/netmito/src/config/client/artifacts.rs +++ b/netmito/src/config/client/artifacts.rs @@ -88,6 +88,10 @@ pub struct DownloadArtifactsByFilterArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite the tasks belong to. Names the group on its + /// own, so `--group` is only needed to double-check it + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -152,6 +156,7 @@ impl From for ArtifactsDownloadByFilterReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { @@ -194,6 +199,10 @@ pub struct DeleteArtifactsByFilterArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite the tasks belong to. Names the group on its + /// own, so `--group` is only needed to double-check it + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -234,6 +243,7 @@ impl From for ArtifactsDeleteByFilterReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { diff --git a/netmito/src/config/client/tasks.rs b/netmito/src/config/client/tasks.rs index 5d45010..7a97587 100644 --- a/netmito/src/config/client/tasks.rs +++ b/netmito/src/config/client/tasks.rs @@ -96,6 +96,10 @@ pub struct QueryTasksArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite the tasks belong to. Names the group on its + /// own, so `--group` is only needed to double-check it + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -142,6 +146,10 @@ pub struct CancelTasksArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, + /// The UUID of the task suite the tasks belong to. Names the group on its + /// own, so `--group` is only needed to double-check it + #[arg(long = "suite")] + pub suite_uuid: Option, /// The tags of the tasks #[arg(short, long, num_args = 0.., value_delimiter = ',')] pub tags: Vec, @@ -215,6 +223,7 @@ impl From for TasksQueryReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { @@ -283,6 +292,7 @@ impl From for TasksCancelByFilterReq { Some(args.creators.into_iter().collect()) }, group_name: args.group, + suite_uuid: args.suite_uuid, tags: if args.tags.is_empty() { None } else { diff --git a/netmito/src/schema/artifact.rs b/netmito/src/schema/artifact.rs index 47f153d..c8fba3f 100644 --- a/netmito/src/schema/artifact.rs +++ b/netmito/src/schema/artifact.rs @@ -32,6 +32,7 @@ pub struct ArtifactQueryResp { pub struct ArtifactsDownloadByFilterReq { pub creator_usernames: Option>, pub group_name: Option, + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, @@ -70,6 +71,7 @@ pub struct ArtifactsDownloadListResp { pub struct ArtifactsDeleteByFilterReq { pub creator_usernames: Option>, pub group_name: Option, + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, diff --git a/netmito/src/schema/task.rs b/netmito/src/schema/task.rs index 56a73cf..0abe9b4 100644 --- a/netmito/src/schema/task.rs +++ b/netmito/src/schema/task.rs @@ -100,6 +100,9 @@ pub struct TaskQueryInfo { pub uuid: Uuid, pub creator_username: String, pub group_name: String, + /// The suite that owns this task, if any. A suite always belongs to + /// `group_name`, since submitting to one pins the task's group to it. + pub suite_uuid: Option, pub task_id: i64, pub tags: Vec, pub labels: Vec, @@ -120,6 +123,7 @@ pub struct ParsedTaskQueryInfo { pub uuid: Uuid, pub creator_username: String, pub group_name: String, + pub suite_uuid: Option, pub task_id: i64, pub tags: Vec, pub labels: Vec, @@ -142,6 +146,10 @@ pub struct ParsedTaskQueryInfo { pub struct TasksQueryReq { pub creator_usernames: Option>, pub group_name: Option, + /// Restrict to the tasks of one suite. The suite names its own group, so it + /// fills in `group_name` when that was left out and contradicts it + /// otherwise. + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, @@ -160,6 +168,7 @@ pub struct TasksQueryReq { pub struct TasksCancelByFilterReq { pub creator_usernames: Option>, pub group_name: Option, + pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, pub states: Option>, diff --git a/netmito/src/service/s3.rs b/netmito/src/service/s3.rs index e56b951..5221f51 100644 --- a/netmito/src/service/s3.rs +++ b/netmito/src/service/s3.rs @@ -253,6 +253,7 @@ pub async fn batch_download_artifacts_by_filter( let query = TasksQueryReq { creator_usernames: req.creator_usernames, group_name: req.group_name, + suite_uuid: req.suite_uuid, tags: req.tags, labels: req.labels, states: req.states, @@ -1322,6 +1323,7 @@ pub async fn batch_delete_artifacts_by_filter( let query = TasksQueryReq { creator_usernames: req.creator_usernames, group_name: req.group_name, + suite_uuid: req.suite_uuid, tags: req.tags, labels: req.labels, states: req.states, diff --git a/netmito/src/service/task.rs b/netmito/src/service/task.rs index e3a806b..eced5ef 100644 --- a/netmito/src/service/task.rs +++ b/netmito/src/service/task.rs @@ -675,6 +675,10 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), + ) .from(ActiveTasks::Entity) .join( sea_orm::JoinType::Join, @@ -690,6 +694,16 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((ActiveTasks::Entity, ActiveTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) + // Left join: a task without a suite is the common case, and those + // rows have to survive it. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ActiveTasks::Entity, + ActiveTasks::Column::TaskSuiteId, + ))), + ) .and_where(Expr::col((ActiveTasks::Entity, ActiveTasks::Column::Uuid)).eq(uuid)) .limit(1) .to_owned(); @@ -724,6 +738,10 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), + ) .from(ArchivedTasks::Entity) .join( sea_orm::JoinType::Join, @@ -739,6 +757,16 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) + // Left join: a task without a suite is the common case, and those + // rows have to survive it. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ArchivedTasks::Entity, + ArchivedTasks::Column::TaskSuiteId, + ))), + ) .and_where(Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::Uuid)).eq(uuid)) .limit(1) .to_owned(); @@ -768,6 +796,7 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res uuid: info.uuid, creator_username: info.creator_username, group_name: info.group_name, + suite_uuid: info.suite_uuid, task_id: info.task_id, tags: info.tags, labels: info.labels, @@ -790,12 +819,21 @@ struct UserGroupRoleQueryRes { role: UserGroupRole, } +/// The suite a query is restricted to, resolved together with the group that +/// owns it so the two can be checked against each other. +#[derive(FromQueryResult)] +struct SuiteIdWithGroup { + id: i64, + group_name: String, +} + +/// Returns the id of the suite the query names, for [`apply_task_filters`]. pub(crate) async fn check_task_list_query( user_id: i64, pool: &InfraPool, query: &mut TasksQueryReq, role: UserGroupRole, -) -> crate::error::Result<()> { +) -> crate::error::Result> { if let Some(ref tags) = query.tags { if tags.is_empty() { return Err(Error::ApiError(crate::error::ApiError::InvalidRequest( @@ -824,6 +862,69 @@ pub(crate) async fn check_task_list_query( ))); } } + // A suite belongs to exactly one group, and submitting to it pins the task + // to that same group, so the suite names the group rather than narrowing + // it: it fills in a group the caller left out, and contradicts one they got + // wrong. Resolved through the caller's own memberships, so a suite in a + // group they are not in, one that does not exist, and one that disagrees + // with `group_name` are all the same answer — and that answer never names + // the owning group, which would otherwise leak out of a bare uuid. + let suite_id = match query.suite_uuid { + None => None, + Some(suite_uuid) => { + let builder = pool.db.get_database_backend(); + let suite_stmt = Query::select() + .column((TaskSuites::Entity, TaskSuites::Column::Id)) + .expr_as( + Expr::col((Group::Entity, Group::Column::GroupName)), + Alias::new("group_name"), + ) + .from(TaskSuites::Entity) + .join( + sea_orm::JoinType::Join, + Group::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::GroupId)) + .eq(Expr::col((Group::Entity, Group::Column::Id))), + ) + .join( + sea_orm::JoinType::Join, + UserGroup::Entity, + Expr::col((UserGroup::Entity, UserGroup::Column::GroupId)) + .eq(Expr::col((TaskSuites::Entity, TaskSuites::Column::GroupId))), + ) + .and_where(Expr::col((UserGroup::Entity, UserGroup::Column::UserId)).eq(user_id)) + .and_where(Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)).eq(suite_uuid)) + .to_owned(); + let suite = SuiteIdWithGroup::find_by_statement(builder.build(&suite_stmt)) + .one(&pool.db) + .await? + .ok_or_else(|| { + // This fires when there is no query result, i.e. the suite does not exist or + // the user does not have access to it + Error::ApiError(crate::error::ApiError::NotFound(format!( + "User doesn't have permission or suite with uuid {suite_uuid}" + ))) + })?; + match query.group_name { + Some(ref group_name) if *group_name != suite.group_name => { + // This fires then the user's specified group_name does not match + // the actual owner group of the suite. + return Err(Error::ApiError(crate::error::ApiError::NotFound(format!( + "User doesn't have permission or suite with uuid {suite_uuid}" + )))); + } + Some(_) => {} + None => { + tracing::debug!( + "No group name specified, use the suite's group {} instead", + suite.group_name + ); + query.group_name = Some(suite.group_name); + } + } + Some(suite.id) + } + }; if query.group_name.is_none() { let username = User::Entity::find() .filter(User::Column::Id.eq(user_id)) @@ -866,7 +967,7 @@ pub(crate) async fn check_task_list_query( } } } - Ok(()) + Ok(suite_id) } #[derive(Debug, Copy, Clone)] @@ -905,7 +1006,19 @@ pub(crate) fn apply_task_filters( active_stmt: &mut sea_orm::sea_query::SelectStatement, archive_stmt: &mut sea_orm::sea_query::SelectStatement, query: &TasksQueryReq, + suite_id: Option, ) -> crate::error::Result<()> { + // The id rather than `query.suite_uuid`: `check_task_list_query` had to read + // the row anyway to authorize it, and the archive keeps the same column, so + // neither statement needs the suite table joined. + if let Some(suite_id) = suite_id { + active_stmt.and_where( + Expr::col((ActiveTasks::Entity, ActiveTasks::Column::TaskSuiteId)).eq(suite_id), + ); + archive_stmt.and_where( + Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::TaskSuiteId)).eq(suite_id), + ); + } if let Some(runner_uuid) = query.runner_uuid { active_stmt.and_where( Expr::col((ActiveTasks::Entity, ActiveTasks::Column::RunnerUuid)).eq(runner_uuid), @@ -1105,7 +1218,7 @@ pub async fn query_tasks_by_filter( pool: &InfraPool, mut query: TasksQueryReq, ) -> crate::error::Result { - check_task_list_query(user_id, pool, &mut query, UserGroupRole::Read).await?; + let suite_id = check_task_list_query(user_id, pool, &mut query, UserGroupRole::Read).await?; let mut active_stmt = Query::select(); if query.count { active_stmt.expr(Expr::col((ActiveTasks::Entity, ActiveTasks::Column::Uuid)).count()); @@ -1134,6 +1247,10 @@ pub async fn query_tasks_by_filter( .expr_as( Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), + ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), ); } active_stmt @@ -1151,6 +1268,16 @@ pub async fn query_tasks_by_filter( Group::Entity, Expr::col((ActiveTasks::Entity, ActiveTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), + ) + // Left join: a task without a suite is the common case, and those + // rows have to survive it. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ActiveTasks::Entity, + ActiveTasks::Column::TaskSuiteId, + ))), ); let mut archive_stmt = Query::select(); if query.count { @@ -1186,6 +1313,10 @@ pub async fn query_tasks_by_filter( .expr_as( Expr::col((Group::Entity, Group::Column::GroupName)), Alias::new("group_name"), + ) + .expr_as( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Uuid)), + Alias::new("suite_uuid"), ); } archive_stmt @@ -1203,10 +1334,20 @@ pub async fn query_tasks_by_filter( Group::Entity, Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), + ) + // Left join: a task without a suite is the common case, and those + // rows have to survive it. + .join( + sea_orm::JoinType::LeftJoin, + TaskSuites::Entity, + Expr::col((TaskSuites::Entity, TaskSuites::Column::Id)).eq(Expr::col(( + ArchivedTasks::Entity, + ArchivedTasks::Column::TaskSuiteId, + ))), ); // Apply filters using the shared helper function - apply_task_filters(&mut active_stmt, &mut archive_stmt, &query)?; + apply_task_filters(&mut active_stmt, &mut archive_stmt, &query, suite_id)?; let builder = pool.db.get_database_backend(); let resp = if query.count { let active_count = CountQuery::find_by_statement(builder.build(&active_stmt)) @@ -1258,6 +1399,7 @@ pub async fn cancel_tasks_by_filter( let mut query = TasksQueryReq { creator_usernames: req.creator_usernames, group_name: req.group_name, + suite_uuid: req.suite_uuid, tags: req.tags, labels: req.labels, runner_uuid: None, @@ -1288,7 +1430,7 @@ pub async fn cancel_tasks_by_filter( }; // Validate query and fill in defaults (also checks Write permission) - check_task_list_query(user_id, pool, &mut query, UserGroupRole::Write).await?; + let suite_id = check_task_list_query(user_id, pool, &mut query, UserGroupRole::Write).await?; let group_name = query.group_name.clone().unwrap_or_default(); // Build task ID subquery with the same filters (avoids parameter limits) @@ -1313,7 +1455,12 @@ pub async fn cancel_tasks_by_filter( // Apply the same filters to the subquery let mut dummy_archive_stmt = Query::select().from(ArchivedTasks::Entity).to_owned(); - apply_task_filters(&mut task_id_subquery, &mut dummy_archive_stmt, &query)?; + apply_task_filters( + &mut task_id_subquery, + &mut dummy_archive_stmt, + &query, + suite_id, + )?; // Build the complete CTE statement before the transaction to avoid lifetime issues diff --git a/openapi.yaml b/openapi.yaml index f73ba3e..5b600f9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -3278,6 +3278,16 @@ components: nullable: true description: Filter by group name example: research-team + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Filter by the task suite. A suite names its own group, so it fills in + group_name when that is omitted and must agree with it otherwise. A + suite that does not exist, is in a group the caller is not in, or + disagrees with group_name is a 404, not an empty result. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -3359,6 +3369,11 @@ components: group_name: type: string example: research-team + suite_uuid: + type: string + format: uuid + nullable: true + description: The task suite that owns this task, if any task_id: type: integer format: int64 @@ -3439,6 +3454,11 @@ components: type: string group_name: type: string + suite_uuid: + type: string + format: uuid + nullable: true + description: The task suite that owns this task, if any task_id: type: integer format: int64 @@ -3677,6 +3697,16 @@ components: type: string nullable: true description: Filter by group name + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Filter by the task suite. A suite names its own group, so it fills in + group_name when that is omitted and must agree with it otherwise. A + suite that does not exist, is in a group the caller is not in, or + disagrees with group_name is a 404, not an empty result. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -3867,6 +3897,16 @@ components: type: string nullable: true description: Filter by group name + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Filter by the task suite. A suite names its own group, so it fills in + group_name when that is omitted and must agree with it otherwise. A + suite that does not exist, is in a group the caller is not in, or + disagrees with group_name is a 404, not an empty result. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: @@ -4045,6 +4085,16 @@ components: type: string nullable: true description: Filter by group name + suite_uuid: + type: string + format: uuid + nullable: true + description: >- + Filter by the task suite. A suite names its own group, so it fills in + group_name when that is omitted and must agree with it otherwise. A + suite that does not exist, is in a group the caller is not in, or + disagrees with group_name is a 404, not an empty result. + example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array items: From f38b474045cef230abbb5c3b153e763be8d44eae Mon Sep 17 00:00:00 2001 From: un-lock-able Date: Fri, 16 Oct 2026 16:59:59 +0800 Subject: [PATCH 2/4] feat(suite): allow changing suite properties --- netmito/src/api/suites.rs | 24 ++- netmito/src/client/http.rs | 26 ++++ netmito/src/client/mod.rs | 16 +- netmito/src/config/client/mod.rs | 16 +- netmito/src/config/client/suites.rs | 150 ++++++++++++++++++- netmito/src/schema/suite.rs | 94 +++++++++++- netmito/src/service/suite.rs | 219 ++++++++++++++++++++++++++-- netmito/src/service/task.rs | 2 +- openapi.yaml | 128 ++++++++++++++++ 9 files changed, 650 insertions(+), 25 deletions(-) diff --git a/netmito/src/api/suites.rs b/netmito/src/api/suites.rs index 149cbb6..dec0314 100644 --- a/netmito/src/api/suites.rs +++ b/netmito/src/api/suites.rs @@ -11,8 +11,9 @@ use crate::{ config::InfraPool, error::ApiError, schema::{ - CancelTaskSuiteParam, CreateTaskSuiteReq, CreateTaskSuiteResp, SuiteAgentOverrideReq, - SuiteAgentOverrideResp, TaskSuiteQueryResp, TaskSuitesQueryReq, TaskSuitesQueryResp, + CancelTaskSuiteParam, ChangeTaskSuiteReq, CreateTaskSuiteReq, CreateTaskSuiteResp, + SuiteAgentOverrideReq, SuiteAgentOverrideResp, TaskSuiteQueryResp, TaskSuitesQueryReq, + TaskSuitesQueryResp, }, service::{ self, @@ -24,7 +25,12 @@ pub fn suites_router(st: InfraPool) -> Router { Router::new() .route("/", post(create_suite)) .route("/query", post(query_suites)) - .route("/{uuid}", get(get_suite_details).delete(cancel_suite)) + .route( + "/{uuid}", + get(get_suite_details) + .put(change_suite) + .delete(cancel_suite), + ) .route("/{uuid}/close", post(close_suite)) .route("/{uuid}/agents/override", post(override_agents_for_suite)) .route_layer(middleware::from_fn_with_state( @@ -67,6 +73,18 @@ pub async fn get_suite_details( Ok(Json(details)) } +pub async fn change_suite( + Extension(u): Extension, + State(pool): State, + Path(uuid): Path, + Json(req): Json, +) -> Result<(), ApiError> { + service::suite::user_change_task_suite(u.id, &pool, uuid, req) + .await + .map_err(map_service_error)?; + Ok(()) +} + pub async fn close_suite( Extension(u): Extension, State(pool): State, diff --git a/netmito/src/client/http.rs b/netmito/src/client/http.rs index 9627780..94e0299 100644 --- a/netmito/src/client/http.rs +++ b/netmito/src/client/http.rs @@ -1680,6 +1680,32 @@ impl MitoHttpClient { } } + pub async fn change_task_suite( + &mut self, + uuid: Uuid, + req: ChangeTaskSuiteReq, + ) -> crate::error::Result<()> { + let credential = self + .credential_guard + .get_credential() + .ok_or(CredentialGuardError::CredentialNotFound)? + .token; + self.url.set_path(&format!("suites/{uuid}")); + let resp = self + .http_client + .put(self.url.as_str()) + .bearer_auth(credential) + .json(&req) + .send() + .await + .map_err(map_reqwest_err)?; + if resp.status().is_success() { + Ok(()) + } else { + Err(get_error_from_resp(resp).await.into()) + } + } + pub async fn close_task_suite(&mut self, uuid: Uuid) -> crate::error::Result<()> { let credential = self .credential_guard diff --git a/netmito/src/client/mod.rs b/netmito/src/client/mod.rs index 097ac20..530198a 100644 --- a/netmito/src/client/mod.rs +++ b/netmito/src/client/mod.rs @@ -820,6 +820,12 @@ impl MitoClient { self.http_client.create_task_suite(args.into()).await } + pub async fn suites_change(&mut self, args: ChangeSuiteArgs) -> crate::error::Result<()> { + self.http_client + .change_task_suite(args.uuid, args.into()) + .await + } + pub async fn suites_query( &mut self, args: QuerySuitesArgs, @@ -2064,7 +2070,7 @@ impl MitoClient { }, }, ClientCommand::Suites(args) => match args.command { - SuitesCommands::Create(args) => match self.suites_create(args).await { + SuitesCommands::Create(args) => match self.suites_create(*args).await { Ok(resp) => { tracing::info!("Suite created with uuid {}", resp.uuid); } @@ -2072,6 +2078,14 @@ impl MitoClient { tracing::error!("{}", e); } }, + SuitesCommands::Change(args) => match self.suites_change(*args).await { + Ok(_) => { + tracing::info!("Suite changed successfully"); + } + Err(e) => { + tracing::error!("{}", e); + } + }, SuitesCommands::Query(args) => { let verbose = args.verbose; let counted = args.count; diff --git a/netmito/src/config/client/mod.rs b/netmito/src/config/client/mod.rs index 0d22191..96e12c0 100644 --- a/netmito/src/config/client/mod.rs +++ b/netmito/src/config/client/mod.rs @@ -9,7 +9,7 @@ use std::ops::Not; use url::Url; use crate::entity::content::ArtifactContentType; -use crate::schema::{RemoteResource, RemoteResourceDownload}; +use crate::schema::{ExecSpec, RemoteResource, RemoteResourceDownload}; use std::path::PathBuf; use uuid::Uuid; @@ -158,6 +158,20 @@ fn parse_artifact_content_type( } } +/// Parse a hook's `ExecSpec` from a JSON object, the same shape `POST /suites` +/// takes: `{"args":["sh","-c","./setup.sh"],"terminal_output":true}`. Only +/// `args` is required. +fn parse_exec_spec( + s: &str, +) -> Result> { + let spec: ExecSpec = serde_json::from_str(s) + .map_err(|e| format!("invalid hook spec: {e}. Expected a JSON object such as {{\"args\":[\"sh\",\"-c\",\"./setup.sh\"],\"terminal_output\":true}}"))?; + if spec.args.is_empty() { + return Err("invalid hook spec: `args` must not be empty".into()); + } + Ok(spec) +} + /// Parse a resource string into RemoteResourceDownload fn parse_resources( s: &str, diff --git a/netmito/src/config/client/suites.rs b/netmito/src/config/client/suites.rs index e18ddc4..ae15fa9 100644 --- a/netmito/src/config/client/suites.rs +++ b/netmito/src/config/client/suites.rs @@ -1,12 +1,19 @@ +use std::collections::HashSet; + use clap::{Args, Subcommand}; use serde::{Deserialize, Serialize}; use uuid::Uuid; use crate::{ entity::state::TaskSuiteState, - schema::{CreateTaskSuiteReq, TaskSuitesQueryReq, WorkerSchedulePlan}, + schema::{ + ChangeTaskSuiteReq, CreateTaskSuiteReq, ExecHooks, ExecHooksUpdate, ExecSpec, + TaskSuitesQueryReq, UpdateOp, WorkerSchedulePlan, + }, }; +use super::parse_exec_spec; + #[derive(Serialize, Debug, Deserialize, Args, derive_more::From, Clone)] pub struct SuitesArgs { #[command(subcommand)] @@ -16,7 +23,10 @@ pub struct SuitesArgs { #[derive(Subcommand, Serialize, Debug, Deserialize, Clone)] pub enum SuitesCommands { /// Create a new task suite - Create(CreateSuiteArgs), + // Boxed: the three optional hook specs make this variant far larger than the rest. + Create(Box), + /// Change a suite's properties. Takes effect on the jobs started after it + Change(Box), /// Query task suites subject to a filter Query(QuerySuitesArgs), /// Get the details of a task suite @@ -55,10 +65,30 @@ pub struct CreateSuiteArgs { /// Number of tasks each worker prefetches locally #[arg(long, default_value_t = 16)] pub prefetch: u32, + /// Provision hook, as a JSON exec spec: '{"args":["sh","-c","./setup.sh"],"terminal_output":true}'. + /// Runs once before any task; a non-zero exit fails the job and its tasks never start + #[arg(long, value_parser = parse_exec_spec)] + pub provision: Option, + /// Cleanup hook, same JSON shape. Runs once after the tasks drain, even when the job already failed + #[arg(long, value_parser = parse_exec_spec)] + pub cleanup: Option, + /// Background hook, same JSON shape. Runs alongside the tasks and is expected to outlive them + #[arg(long, value_parser = parse_exec_spec)] + pub background: Option, } impl From for CreateTaskSuiteReq { fn from(args: CreateSuiteArgs) -> Self { + // A suite with no hook at all keeps `exec_hooks` absent rather than + // carrying three nulls. + let exec_hooks = match (args.provision, args.cleanup, args.background) { + (None, None, None) => None, + (provision, cleanup, background) => Some(ExecHooks { + provision, + cleanup, + background, + }), + }; Self { name: args.name, description: args.description, @@ -71,7 +101,121 @@ impl From for CreateTaskSuiteReq { cpu_binding: None, task_prefetch_count: args.prefetch, }, - exec_hooks: None, + exec_hooks, + } + } +} + +/// Every field is left alone unless named. Each nullable one has a `--clear-*` +/// partner that conflicts with its value flag, so setting and clearing at once +/// is a parse error rather than a rejected request. +#[derive(Serialize, Debug, Deserialize, Args, Clone)] +pub struct ChangeSuiteArgs { + /// The UUID of the suite + pub uuid: Uuid, + /// New human-readable name for the suite + #[arg(short, long, conflicts_with = "clear_name")] + pub name: Option, + /// Remove the suite's name + #[arg(long)] + pub clear_name: bool, + /// New description for the suite + #[arg(short, long, conflicts_with = "clear_description")] + pub description: Option, + /// Remove the suite's description + #[arg(long)] + pub clear_description: bool, + /// Replacement tags for agent matching (e.g. gpu,linux) + #[arg(short, long, num_args = 1.., value_delimiter = ',', conflicts_with = "clear_tags")] + pub tags: Option>, + /// Drop every tag. An untagged suite matches every agent its group can write to + #[arg(long)] + pub clear_tags: bool, + /// Replacement labels for querying/filtering + #[arg(short, long, num_args = 1.., value_delimiter = ',', conflicts_with = "clear_labels")] + pub labels: Option>, + /// Drop every label + #[arg(long)] + pub clear_labels: bool, + /// Suite scheduling priority (higher = more important) + #[arg(short, long)] + pub priority: Option, + /// Number of workers each agent spawns for this suite. Must specify prefetch + /// setting alongside + #[arg(short, long, requires = "prefetch")] + pub workers: Option, + /// Number of tasks each worker prefetches locally. Must specify worker count + /// settings alongside + #[arg(long, requires = "workers")] + pub prefetch: Option, + /// Replacement provision hook, as a JSON exec spec + #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_provision")] + pub provision: Option, + /// Remove the provision hook + #[arg(long)] + pub clear_provision: bool, + /// Replacement cleanup hook, same JSON shape + #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_cleanup")] + pub cleanup: Option, + /// Remove the cleanup hook + #[arg(long)] + pub clear_cleanup: bool, + /// Replacement background hook, same JSON shape + #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_background")] + pub background: Option, + /// Remove the background hook + #[arg(long)] + pub clear_background: bool, +} + +/// A value flag and its `--clear-*` partner, as the one op they describe. +/// clap rules out the both-given case, so a value wins if one ever arrives. +fn update_op(value: Option, clear: bool) -> Option> { + match (value, clear) { + (Some(value), _) => Some(UpdateOp::SetTo(value)), + (None, true) => Some(UpdateOp::SetNull), + (None, false) => None, + } +} + +impl From for ChangeTaskSuiteReq { + fn from(args: ChangeSuiteArgs) -> Self { + // An array column has no null, so its cleared state is the empty set — + // the flag pair is uniform at the CLI even where the request is not. + let tags = match (args.tags, args.clear_tags) { + (Some(tags), _) => Some(tags.into_iter().collect()), + (None, true) => Some(HashSet::new()), + (None, false) => None, + }; + let labels = match (args.labels, args.clear_labels) { + (Some(labels), _) => Some(labels.into_iter().collect()), + (None, true) => Some(HashSet::new()), + (None, false) => None, + }; + // The plan is replaced wholesale, so it is stated in full or not at all: + // clap requires the two flags together, and neither half may default over + // what the suite already had. + let worker_schedule = match (args.workers, args.prefetch) { + (Some(worker_count), Some(prefetch)) => Some(WorkerSchedulePlan::FixedWorkers { + worker_count, + cpu_binding: None, + task_prefetch_count: prefetch, + }), + _ => None, + }; + let hooks = ExecHooksUpdate { + provision: update_op(args.provision, args.clear_provision), + cleanup: update_op(args.cleanup, args.clear_cleanup), + background: update_op(args.background, args.clear_background), + }; + Self { + name: update_op(args.name, args.clear_name), + description: update_op(args.description, args.clear_description), + tags, + labels, + priority: args.priority, + worker_schedule, + exec_hooks: (!hooks.is_empty()).then_some(hooks), } } } diff --git a/netmito/src/schema/suite.rs b/netmito/src/schema/suite.rs index d97a916..e400eeb 100644 --- a/netmito/src/schema/suite.rs +++ b/netmito/src/schema/suite.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use crate::entity::state::TaskSuiteState; -use super::exec::ExecHooks; +use super::exec::{ExecHooks, ExecSpec}; /// Request to create a new task suite #[derive(Debug, Clone, Serialize, Deserialize)] @@ -43,6 +43,98 @@ pub struct CreateTaskSuiteResp { pub uuid: Uuid, } +/// A change to a field whose column is nullable. The field itself is wrapped in +/// an `Option`, where absent means "leave alone" — so the two variants are the +/// only things a caller can ask for, and neither can be confused with silence. +/// +/// Fields backed by a `NOT NULL` column keep a plain `Option`: they have no null +/// to set, and their cleared state is a value (an empty array, say). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "op", content = "value", rename_all = "snake_case")] +pub enum UpdateOp { + SetTo(T), + SetNull, +} + +impl UpdateOp { + /// The value this op stores. + pub fn into_value(self) -> Option { + match self { + Self::SetTo(value) => Some(value), + Self::SetNull => None, + } + } +} + +/// Per-hook changes to a suite's `exec_hooks`. Each hook is set, cleared, or — +/// absent — left as it was; the alternative, replacing the whole object, would +/// drop the hooks a caller merely failed to mention. Clearing all three stores +/// no hooks at all, the shape creation leaves behind for a suite with none. +/// +/// The merge itself is done in SQL, over the column's own value, so two +/// concurrent patches cannot lose each other's edit. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ExecHooksUpdate { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provision: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cleanup: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub background: Option>, +} + +impl ExecHooksUpdate { + pub fn is_empty(&self) -> bool { + self.provision.is_none() && self.cleanup.is_none() && self.background.is_none() + } +} + +/// Request to change a suite's properties. Every field is absent-means-unchanged, +/// and an empty request is rejected rather than treated as a no-op. +/// +/// A change reaches the jobs started after it, never the ones already running: +/// slot counts and hooks are snapshotted when a job starts, and an agent that +/// stops matching the new tags still finishes what it holds. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ChangeTaskSuiteReq { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option>, + /// Tags for agent matching. An empty set matches every agent the suite's + /// group can write to, since matching is array containment. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub labels: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub priority: Option, + /// Replaces the plan wholesale — a patch could not express switching to + /// another scheduling policy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_schedule: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exec_hooks: Option, +} + +impl ChangeTaskSuiteReq { + pub fn is_empty(&self) -> bool { + self.name.is_none() + && self.description.is_none() + && self.tags.is_none() + && self.labels.is_none() + && self.priority.is_none() + && self.worker_schedule.is_none() + && self.exec_hooks.as_ref().map_or(true, |h| h.is_empty()) + } + + /// Whether the change touches what the scheduler reads: who may run the + /// suite, and where it sits in the pick order. + pub fn affects_scheduling(&self) -> bool { + self.tags.is_some() || self.priority.is_some() + } +} + /// Worker scheduling policy for the suite /// This enum allows for future extension with different scheduling strategies #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/netmito/src/service/suite.rs b/netmito/src/service/suite.rs index d68ae44..c8b73fc 100644 --- a/netmito/src/service/suite.rs +++ b/netmito/src/service/suite.rs @@ -17,18 +17,48 @@ use crate::entity::{ }; use crate::error::{ApiError, Error, ResolveError, Result}; use crate::schema::{ - CancelTaskSuiteOp, CountQuery, CreateTaskSuiteReq, CreateTaskSuiteResp, ExecHooks, - ParsedTaskSuiteInfo, SuiteAgentOverrideReq, SuiteAgentOverrideResp, TaskResultMessage, - TaskResultSpec, TaskSuiteInfo, TaskSuiteQueryResp, TaskSuitesQueryReq, TaskSuitesQueryResp, - WorkerSchedulePlan, + CancelTaskSuiteOp, ChangeTaskSuiteReq, CountQuery, CreateTaskSuiteReq, CreateTaskSuiteResp, + ExecHooks, ParsedTaskSuiteInfo, SuiteAgentOverrideReq, SuiteAgentOverrideResp, + TaskResultMessage, TaskResultSpec, TaskSuiteInfo, TaskSuiteQueryResp, TaskSuitesQueryReq, + TaskSuitesQueryResp, UpdateOp, WorkerSchedulePlan, }; -use crate::service::task::{parse_operators_with_number, OperatorWithNumber}; +use crate::service::task::{check_exec_spec, parse_operators_with_number, OperatorWithNumber}; #[derive(FromQueryResult)] struct GroupIdResult { id: i64, } +/// The largest worker count a plan may ask an agent for. +const MAX_WORKER_COUNT: u32 = 256; + +/// Reject a plan no agent should be handed. Shared with the change path so an +/// edit cannot reach a state creation refuses. +// TODO: this should finally be adjusted to some more flexible definitions +fn validate_worker_schedule(plan: &WorkerSchedulePlan) -> Result<()> { + match plan { + WorkerSchedulePlan::FixedWorkers { worker_count, .. } => { + if *worker_count == 0 || *worker_count > MAX_WORKER_COUNT { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "worker_count must be between 1 and {MAX_WORKER_COUNT}, got {worker_count}" + )))); + } + } + } + Ok(()) +} + +/// Hooks run through the same executor as tasks, so their specs face the same check. +fn validate_exec_hooks(hooks: &ExecHooks) -> Result<()> { + for spec in [&hooks.provision, &hooks.cleanup, &hooks.background] + .into_iter() + .flatten() + { + check_exec_spec(spec)?; + } + Ok(()) +} + pub async fn user_create_task_suite( user_id: i64, pool: &InfraPool, @@ -43,16 +73,9 @@ pub async fn user_create_task_suite( exec_hooks, }: CreateTaskSuiteReq, ) -> crate::error::Result { - // Validate the worker schedule based on the policy variant. - match &worker_schedule { - // TODO: this should finally be adjusted to some more flexible definitions - WorkerSchedulePlan::FixedWorkers { - worker_count: _worker_count, - task_prefetch_count: _task_prefetch_count, - .. - } => { - // TODO: validate the request - } + validate_worker_schedule(&worker_schedule)?; + if let Some(hooks) = &exec_hooks { + validate_exec_hooks(hooks)?; } if group_name.is_empty() { @@ -211,6 +234,162 @@ pub(crate) async fn check_task_suites_query( Ok(()) } +/// Change a suite's properties. Requires Write in the suite's group. +/// +/// A `Cancelled` suite is frozen — effectively archived, so even a rename is +/// rejected. Every other state is editable. +/// +/// The change reaches the *next* job. Slot counts and hooks are snapshotted when +/// a job starts, and an agent that stops tag-matching continue to run the job. +pub async fn user_change_task_suite( + user_id: i64, + pool: &InfraPool, + suite_uuid: Uuid, + req: ChangeTaskSuiteReq, +) -> Result<()> { + if req.is_empty() { + return Err(Error::ApiError(ApiError::InvalidRequest( + "No change specified".to_string(), + ))); + } + if let Some(plan) = &req.worker_schedule { + validate_worker_schedule(plan)?; + } + // Only the incoming specs need checking: the hooks left alone were checked + // when they were written. + if let Some(update) = &req.exec_hooks { + for op in [&update.provision, &update.cleanup, &update.background] + .into_iter() + .flatten() + { + if let UpdateOp::SetTo(spec) = op { + check_exec_spec(spec)?; + } + } + } + let _needs_notify = req.affects_scheduling(); + let now = TimeDateTimeWithTimeZone::now_utc(); + + let _suite_id = pool + .db + .transaction::<_, i64, Error>(|txn| { + Box::pin(async move { + let suite = + match authorize_suite(txn, user_id, suite_uuid, UserGroupRole::Write).await { + Ok(suite) => suite, + Err(ResolveError::Item(e)) => { + return Err(Error::ApiError(ApiError::NotFound(e.msg))) + } + Err(ResolveError::Fatal(e)) => return Err(e), + }; + let suite_id = suite.id; + + // `authorize_suite` takes no lock, so this only names the state + // in the error; the `WHERE` below is what rejects a suite + // cancelled in the meantime. + if suite.state == TaskSuiteState::Cancelled { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Cannot change a {} suite", + suite.state + )))); + } + + let mut change = TaskSuites::ActiveModel { + updated_at: Set(now), + ..Default::default() + }; + if let Some(op) = req.name { + change.name = Set(op.into_value()); + } + if let Some(op) = req.description { + change.description = Set(op.into_value()); + } + if let Some(tags) = req.tags { + change.tags = Set(Vec::from_iter(tags)); + } + if let Some(labels) = req.labels { + change.labels = Set(Vec::from_iter(labels)); + } + if let Some(priority) = req.priority { + change.priority = Set(priority); + } + if let Some(plan) = req.worker_schedule { + change.worker_schedule = Set(serde_json::to_value(plan)?); + } + // The hooks are a patch onto what the suite already carries, + // and merging them in Rust would mean reading the row first — + // two statements, no lock between them, and two concurrent + // patches silently losing one edit. The column is written as an + // expression over its own current value instead, so the merge + // happens inside the `UPDATE` and there is nothing to read. + let hooks_expr = match req.exec_hooks { + Some(update) => { + // A NULL column merges as `{}`, so a first hook lands on + // a suite that has none. + let mut hooks = Expr::cust_with_expr( + "coalesce($1, '{}'::jsonb)", + Expr::col(TaskSuites::Column::ExecHooks), + ); + for (key, op) in [ + ("provision", update.provision), + ("cleanup", update.cleanup), + ("background", update.background), + ] { + hooks = match op { + // `||` replaces this one key and leaves the rest. + // Each step parenthesises itself: `-` binds + // tighter than `||`, so an unwrapped chain would + // read as `x || (obj - 'cleanup')`. + Some(UpdateOp::SetTo(spec)) => Expr::cust_with_exprs( + "($1 || jsonb_build_object($2::text, $3::jsonb))", + [ + hooks, + Expr::val(key).into(), + Expr::val(serde_json::to_value(spec)?).into(), + ], + ), + Some(UpdateOp::SetNull) => Expr::cust_with_exprs( + "($1 - $2::text)", + [hooks, Expr::val(key).into()], + ), + None => hooks, + }; + } + // Every hook gone is no hooks at all — the shape `create` + // stores, rather than an object of three nulls. + Some(Expr::cust_with_expr("nullif($1, '{}'::jsonb)", hooks)) + } + None => None, + }; + + // `exec_hooks` is not part of `change`, so it is assigned here + // rather than twice: two assignments to one column is an error. + let mut stmt = TaskSuites::Entity::update_many() + .set(change) + .filter(TaskSuites::Column::Id.eq(suite_id)) + .filter(TaskSuites::Column::State.ne(TaskSuiteState::Cancelled)); + if let Some(expr) = hooks_expr { + stmt = stmt.col_expr(TaskSuites::Column::ExecHooks, expr); + } + let changed = stmt.exec(txn).await?; + if changed.rows_affected == 0 { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Suite {suite_uuid} was cancelled" + )))); + } + Ok(suite_id) + }) + }) + .await?; + + // Eligibility and the pick order are read from the row on every query, so a + // change leaves nothing to rebuild — the agents it lets in just have to hear + // that the suite is there. + // TODO: once agents can be notified, `if _needs_notify` call + // `notify_suite_available(pool, _suite_id)` here. + Ok(()) +} + /// Query suites subject to a filter. Only returns suites in a group the caller can read. pub async fn user_query_task_suites( user_id: i64, @@ -330,6 +509,16 @@ pub async fn user_query_task_suites( } } } + if !query.count { + stmt.order_by_expr( + Expr::col((TaskSuites::Entity, TaskSuites::Column::Priority)).into(), + sea_orm::Order::Desc, + ) + .order_by_expr( + Expr::col((TaskSuites::Entity, TaskSuites::Column::CreatedAt)).into(), + sea_orm::Order::Asc, + ); + } if let Some(limit) = query.limit { stmt.limit(limit); } diff --git a/netmito/src/service/task.rs b/netmito/src/service/task.rs index eced5ef..b989cee 100644 --- a/netmito/src/service/task.rs +++ b/netmito/src/service/task.rs @@ -31,7 +31,7 @@ use super::worker::{remove_task, TaskDispatcherOp}; // XXX: Not sure if we can relax the constrains on local path checking. // We currently only check if the path is absolute or contains `..` and not check for `.`. -fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { +pub(crate) fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { if spec.resources.iter().any(|r| { r.local_path.is_absolute() || r.local_path diff --git a/openapi.yaml b/openapi.yaml index 5b600f9..7f7bab9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1644,6 +1644,48 @@ paths: "404": $ref: "#/components/responses/NotFound" + put: + summary: Change task suite + description: >- + Change a suite's properties. Every field is optional and an omitted one is + left unchanged; an empty request is rejected. Nullable fields (name, + description and each hook) take an update op rather than a bare value, so + clearing one is expressible. A Cancelled suite is frozen and rejects every + change, including a rename. The change applies to the jobs started after + it: an agent mid-job keeps the worker count and hooks it started with, and + one that no longer tag-matches still finishes the job it holds. The caller + must have the Write or Admin role in the suite's group; a non-member and an + unknown uuid are indistinguishable and both return 404. + operationId: changeTaskSuite + tags: + - Suites + security: + - bearerAuth: [] + parameters: + - name: uuid + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ChangeTaskSuiteReq" + responses: + "200": + description: Suite changed successfully + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + delete: summary: Cancel task suite description: Cancel a task suite (terminal). Archives all non-terminal tasks of the suite as Cancelled. @@ -4420,6 +4462,92 @@ components: exec_hooks: $ref: "#/components/schemas/ExecHooks" + UpdateOpString: + type: object + description: >- + A change to a nullable string field. `set_to` writes the value, `set_null` + clears it; omitting the field from the request leaves it unchanged. + required: + - op + properties: + op: + type: string + enum: + - set_to + - set_null + value: + type: string + description: Required when op is set_to, absent otherwise + example: + op: set_to + value: nightly-eval + + UpdateOpExecSpec: + type: object + description: >- + A change to one lifecycle hook. `set_to` installs the spec, `set_null` + removes the hook; omitting the field leaves that hook as it was. + required: + - op + properties: + op: + type: string + enum: + - set_to + - set_null + value: + type: object + description: Execution spec, required when op is set_to + + ExecHooksUpdate: + type: object + description: >- + Per-hook changes. Each hook is set, cleared, or — omitted — left alone; + clearing all three stores no hooks at all. + properties: + provision: + $ref: "#/components/schemas/UpdateOpExecSpec" + cleanup: + $ref: "#/components/schemas/UpdateOpExecSpec" + background: + $ref: "#/components/schemas/UpdateOpExecSpec" + + ChangeTaskSuiteReq: + type: object + description: At least one field must be present, or the request is rejected as empty + properties: + name: + $ref: "#/components/schemas/UpdateOpString" + description: + $ref: "#/components/schemas/UpdateOpString" + tags: + type: array + items: + type: string + description: >- + Replacement tags for agent matching. An empty array matches every + agent the suite's group can write to, since matching is containment + example: + - linux + - cuda:11 + labels: + type: array + items: + type: string + description: Replacement labels for querying/filtering + priority: + type: integer + format: int32 + description: Suite scheduling priority (higher = more important) + worker_schedule: + allOf: + - $ref: "#/components/schemas/WorkerSchedulePlan" + description: >- + Replaces the plan wholesale; a patch could not express switching to + another scheduling policy + exec_hooks: + $ref: "#/components/schemas/ExecHooksUpdate" + CreateTaskSuiteResp: type: object required: From 74f7d6ae1d1c80350222d4868307a663a62fef58 Mon Sep 17 00:00:00 2001 From: un-lock-able Date: Fri, 16 Oct 2026 17:30:06 +0800 Subject: [PATCH 3/4] fix(suite): keep suite counters and lock order correct on cancel and close --- netmito/src/service/mod.rs | 22 ++ netmito/src/service/suite.rs | 311 ++++++++++++++---- netmito/src/service/task.rs | 515 ++++++++++++++++++++++++------ netmito/src/service/worker/mod.rs | 56 ++-- 4 files changed, 712 insertions(+), 192 deletions(-) diff --git a/netmito/src/service/mod.rs b/netmito/src/service/mod.rs index c7b4edd..c3aabac 100644 --- a/netmito/src/service/mod.rs +++ b/netmito/src/service/mod.rs @@ -1,3 +1,25 @@ +//! # Lock order: suite before task +//! +//! A transaction that writes both a `task_suites` row and `active_tasks` rows +//! must write the suite row first. +//! +//! Row locks taken by an `UPDATE` or `DELETE` live until the transaction +//! commits, not until the statement ends — so single-statement writes do not +//! make ordering someone else's problem. A transaction accumulates its locks, +//! and two of them taking the same pair in opposite orders deadlock, after a +//! full `deadlock_timeout` of waiting first. +//! +//! Two shapes satisfy it: +//! +//! - The suite id is known up front, as on every commit and single-task +//! cancel: write the suite row first and let the rest follow. +//! - The suite ids are only derivable from the task rows, as on a batch +//! cancel: lock them in the same statement that removes the tasks, and gate +//! the removal on the lock, so no task is taken before its suite is. +//! +//! Touching task rows without writing a suite row is exempt: `claim_candidates` +//! holds `FOR UPDATE SKIP LOCKED` on tasks alone, and nothing waits on it. + pub mod auth; pub mod group; pub mod s3; diff --git a/netmito/src/service/suite.rs b/netmito/src/service/suite.rs index c8b73fc..3b7aa94 100644 --- a/netmito/src/service/suite.rs +++ b/netmito/src/service/suite.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use sea_orm::sea_query::extension::postgres::PgExpr; -use sea_orm::sea_query::{Alias, PgFunc, Query}; +use sea_orm::sea_query::{ + Alias, CommonTableExpression, DeleteStatement, InsertStatement, PgFunc, Query, WithClause, +}; use sea_orm::{prelude::*, ConnectionTrait, FromQueryResult, Set, TransactionTrait}; use uuid::Uuid; @@ -29,6 +31,84 @@ struct GroupIdResult { id: i64, } +#[derive(FromQueryResult)] +struct TaskUuidResult { + uuid: Uuid, +} + +/// Hand `count` finished tasks back to a suite: tick `incomplete_tasks` down, +/// and complete the suite if that empties an already-`Closed` one. +/// +/// Every path that takes an active task out of a suite ends here. A path that +/// archives a suite task without calling this parks the +/// suite short of `Complete` and leaves agents cycling over work that is no +/// longer there. +/// +/// The subtraction is deliberately unguarded. Every caller has just removed +/// exactly `count` rows from `active_tasks`, and each of those rows added one to +/// the counter when it was submitted, so the arithmetic cannot legitimately go +/// below zero. Clamping it would only turn a bug elsewhere into a suite that +/// looks fine; a negative counter is reported instead, loudly. +/// +/// Draining an `Open` suite does not complete it: it stays `Open` until it has +/// also been quiet for `suite_auto_close_timeout`, which is what lets the agent +/// hold its job — and its provisioned environment — open for a late arrival. +/// [`sweep_inactive_suites`] settles it afterwards. A `Closed` suite has no such +/// window to wait out, so it is completed here. +/// +/// The suite row must be taken before any task row, either by this call or by +/// the caller before it — see the lock-order note on [`crate::service`]. +pub(crate) async fn decrement_incomplete_tasks( + db: &C, + suite_id: i64, + count: i32, + now: TimeDateTimeWithTimeZone, +) -> Result<()> { + if count <= 0 { + return Ok(()); + } + + let updated = TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::IncompleteTasks, + Expr::col(TaskSuites::Column::IncompleteTasks).sub(count), + ) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .filter(TaskSuites::Column::Id.eq(suite_id)) + .exec_with_returning(db) + .await?; + + // Nothing here can repair it, and swallowing it would leave a suite that + // never completes and never gets claimed, with no trace of why. + for suite in updated.iter().filter(|s| s.incomplete_tasks < 0) { + tracing::error!( + suite_id = suite.id, + incomplete_tasks = suite.incomplete_tasks, + decremented = count, + "A suite's incomplete_tasks went negative: more tasks were taken off it than were ever counted" + ); + } + + // Conditional on the state and the count, which is what makes a second + // statement safe without a transaction around the pair: a submission racing + // it either lands first and leaves a non-zero count, or lands after and sets + // `Open` itself. + TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Complete), + ) + .col_expr(TaskSuites::Column::CompletedAt, Expr::value(Some(now))) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .filter(TaskSuites::Column::Id.eq(suite_id)) + .filter(TaskSuites::Column::State.eq(TaskSuiteState::Closed)) + .filter(TaskSuites::Column::IncompleteTasks.eq(0)) + .exec(db) + .await?; + + Ok(()) +} + /// The largest worker count a plan may ask an agent for. const MAX_WORKER_COUNT: u32 = 256; @@ -725,6 +805,9 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U Err(ResolveError::Fatal(e)) => return Err(e), }; + // `authorize_suite` takes no lock, so this only names the state + // in the error; the `WHERE` below is what actually rejects a + // second close. if suite.state != TaskSuiteState::Open { return Err(Error::ApiError(ApiError::InvalidRequest(format!( "Cannot transition from {} to Closed", @@ -732,10 +815,21 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U )))); } - let mut suite: TaskSuites::ActiveModel = suite.into(); - suite.state = Set(TaskSuiteState::Closed); - suite.updated_at = Set(now); - suite.update(txn).await?; + let closed = TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Closed), + ) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .filter(TaskSuites::Column::Id.eq(suite.id)) + .filter(TaskSuites::Column::State.eq(TaskSuiteState::Open)) + .exec(txn) + .await?; + if closed.rows_affected == 0 { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Suite {suite_uuid} is no longer Open" + )))); + } Ok(()) }) }) @@ -758,9 +852,11 @@ pub async fn user_cancel_task_suite( _op: CancelTaskSuiteOp, ) -> Result<()> { let now = TimeDateTimeWithTimeZone::now_utc(); + let builder = pool.db.get_database_backend(); - pool.db - .transaction::<_, u64, Error>(|txn| { + let (_suite_id, _cancelled_task_uuids) = pool + .db + .transaction::<_, (i64, Vec), Error>(|txn| { Box::pin(async move { // Resolve the suite and authorize the caller's Write role in one join. let suite = @@ -778,6 +874,35 @@ pub async fn user_cancel_task_suite( )))); } + // The suite is cancelled *before* its tasks are archived, not + // after. This statement locks the suite row, and every submit + // bumps that same row under a `state <> Cancelled` predicate — so + // once this commits, a concurrent submit either got in ahead of + // the delete below or is rejected outright. Write the suite last + // instead and a submit landing between the delete and the write + // leaves a live task in a cancelled suite that nothing will ever + // claim, with `incomplete_tasks` already zeroed. + // + // Zero is exact for the same reason: the delete below takes every + // active row of the suite, and nothing can add another. + let fenced = TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Cancelled), + ) + .col_expr(TaskSuites::Column::IncompleteTasks, Expr::value(0)) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .col_expr(TaskSuites::Column::CompletedAt, Expr::value(Some(now))) + .filter(TaskSuites::Column::Id.eq(suite.id)) + .filter(TaskSuites::Column::State.ne(TaskSuiteState::Cancelled)) + .exec(txn) + .await?; + if fenced.rows_affected == 0 { + return Err(Error::ApiError(ApiError::InvalidRequest(format!( + "Suite {suite_uuid} is already in cancelled state" + )))); + } + // All cancelled tasks share this result. let result = serde_json::to_value(TaskResultSpec { exit_status: 0, @@ -785,73 +910,127 @@ pub async fn user_cancel_task_suite( }) .inspect_err(|e| tracing::error!("{}", e))?; - // Archive every non-terminal task of the suite as Cancelled: - let tasks = ActiveTasks::Entity::delete_many() - .filter(ActiveTasks::Column::TaskSuiteId.eq(suite.id)) - .filter( - ActiveTasks::Column::State - .is_not_in([TaskState::Cancelled, TaskState::Unknown]), + // Avoid returning the rows so that we won't trigger psql's too many + // argument error when archiving many tasks. + // + // `Cancelled` rows are taken deliberately: that state means an + // agent reported `Cancel` and has not yet sent the `Commit` that + // would archive the task. Leaving them behind stalls them in + // `active_tasks` forever — the `Commit` is about to be refused by + // a terminal job, no other agent can claim a non-`Ready` task, and + // nothing reclaims that state. + let delete_stmt = DeleteStatement::new() + .from_table(ActiveTasks::Entity) + .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).eq(suite.id)) + .and_where( + Expr::col(ActiveTasks::Column::State).is_not_in([TaskState::Unknown]), ) - .exec_with_returning(txn) - .await?; + .returning_all() + .to_owned(); + let deleted_cte = CommonTableExpression::new() + .query(delete_stmt) + .table_name(Alias::new("deleted")) + .to_owned(); - // The inflight subset is the only set tied to an executing agent: each - // carries the `runner_uuid` of the agent running it. A Force cancel must - // signal those agents to stop so they don't commit tasks we just archived. - // Collected now and intentionally left unused. - // TODO: wire this to a per-agent shutdown/TasksCancelled - // push once the agent/notification layer exists - let _inflight_signals: Vec<(Option, Uuid)> = tasks - .iter() - .filter(|t| matches!(t.state, TaskState::Running | TaskState::Finished)) - .map(|t| (t.runner_uuid, t.uuid)) - .collect(); - - let cancelled_count = tasks.len() as u64; - if !tasks.is_empty() { - let archived: Vec = tasks - .into_iter() - .map(|task| ArchivedTasks::ActiveModel { - id: Set(task.id), - creator_id: Set(task.creator_id), - group_id: Set(task.group_id), - task_id: Set(task.task_id), - uuid: Set(task.uuid), - tags: Set(task.tags), - labels: Set(task.labels), - created_at: Set(task.created_at), - updated_at: Set(now), - state: Set(TaskState::Cancelled), - runner_uuid: Set(task.runner_uuid), - priority: Set(task.priority), - spec: Set(task.spec), - exec_options: Set(task.exec_options), - result: Set(Some(result.clone())), - upstream_task_uuid: Set(task.upstream_task_uuid), - downstream_task_uuid: Set(task.downstream_task_uuid), - task_suite_id: Set(task.task_suite_id), - }) - .collect(); - - ArchivedTasks::Entity::insert_many(archived) - .exec(txn) - .await?; - } + // The archived copy: every column read out of `deleted` by name, + // with `updated_at`, `state` and `result` overwritten by three + // values bound once for the whole statement. + let select_from_deleted = Query::select() + .expr(Expr::col(Alias::new("id"))) + .expr(Expr::col(Alias::new("creator_id"))) + .expr(Expr::col(Alias::new("group_id"))) + .expr(Expr::col(Alias::new("task_id"))) + .expr(Expr::col(Alias::new("uuid"))) + .expr(Expr::col(Alias::new("tags"))) + .expr(Expr::col(Alias::new("labels"))) + .expr(Expr::col(Alias::new("created_at"))) + .expr(Expr::value(now)) + .expr(Expr::value(TaskState::Cancelled)) + .expr(Expr::col(Alias::new("runner_uuid"))) + .expr(Expr::col(Alias::new("exec_options"))) + .expr(Expr::col(Alias::new("priority"))) + .expr(Expr::col(Alias::new("spec"))) + .expr(Expr::value(result)) + .expr(Expr::col(Alias::new("upstream_task_uuid"))) + .expr(Expr::col(Alias::new("downstream_task_uuid"))) + .expr(Expr::col(Alias::new("task_suite_id"))) + .from(Alias::new("deleted")) + .to_owned(); + + let mut insert_stmt = InsertStatement::new() + .into_table(ArchivedTasks::Entity) + .columns([ + ArchivedTasks::Column::Id, + ArchivedTasks::Column::CreatorId, + ArchivedTasks::Column::GroupId, + ArchivedTasks::Column::TaskId, + ArchivedTasks::Column::Uuid, + ArchivedTasks::Column::Tags, + ArchivedTasks::Column::Labels, + ArchivedTasks::Column::CreatedAt, + ArchivedTasks::Column::UpdatedAt, + ArchivedTasks::Column::State, + ArchivedTasks::Column::RunnerUuid, + ArchivedTasks::Column::ExecOptions, + ArchivedTasks::Column::Priority, + ArchivedTasks::Column::Spec, + ArchivedTasks::Column::Result, + ArchivedTasks::Column::UpstreamTaskUuid, + ArchivedTasks::Column::DownstreamTaskUuid, + // Without this the archived copy of a suite task loses + // its suite. + ArchivedTasks::Column::TaskSuiteId, + ]) + .to_owned(); + // Infallible here: the column list and the select have the same + // arity, which is a property of this code, not of the request. + insert_stmt.select_from(select_from_deleted).unwrap(); + let archived_cte = CommonTableExpression::new() + .query(insert_stmt) + .table_name(Alias::new("archived")) + .to_owned(); - let mut suite: TaskSuites::ActiveModel = suite.into(); - suite.state = Set(TaskSuiteState::Cancelled); - suite.updated_at = Set(now); - suite.completed_at = Set(Some(now)); - suite.update(txn).await?; + // The in-flight subset is the only one an agent is holding right + // now. Those agents are told which task uuids vanished, so they + // stop rather than reporting against rows we just archived. + // + // This reads `deleted`, not `archived`, because it needs each + // task's state as it stood *before* the archive rewrote it to + // `Cancelled`. `archived` is never referenced, and does not have + // to be: Postgres runs a data-modifying CTE exactly once and to + // completion whether or not the primary query reads its output. + let in_flight_stmt = Query::select() + .expr(Expr::col(Alias::new("uuid"))) + .from(Alias::new("deleted")) + .and_where(Expr::col(Alias::new("state")).is_in([ + TaskState::Running, + TaskState::Finished, + TaskState::Cancelled, + ])) + .to_owned(); - Ok(cancelled_count) + let with_ctes = WithClause::new() + .cte(deleted_cte) + .cte(archived_cte) + .to_owned(); + let cancelled_task_uuids: Vec = TaskUuidResult::find_by_statement( + builder.build(&in_flight_stmt.with(with_ctes)), + ) + .all(txn) + .await? + .into_iter() + .map(|t| t.uuid) + .collect(); + + let suite_id = suite.id; + Ok((suite_id, cancelled_task_uuids)) }) }) .await?; - // TODO: notify assigned/executing agents that this suite was cancelled (and, for - // Force, push per-agent TasksCancelled built from `_running_signals` above), then - // drop the suite's in-memory task buffer in the dispatcher. + // TODO: once agents run suites, stop the jobs on `_suite_id` (`Force` kills + // them, `Graceful` lets them drain) and send each agent `SuiteCancelled`, plus + // `TasksCancelled` for `_cancelled_task_uuids`. Ok(()) } diff --git a/netmito/src/service/task.rs b/netmito/src/service/task.rs index b989cee..fb4d740 100644 --- a/netmito/src/service/task.rs +++ b/netmito/src/service/task.rs @@ -1,10 +1,10 @@ use sea_orm::sea_query::{ extension::postgres::PgExpr, Alias, CommonTableExpression, DeleteStatement, ExprTrait, - InsertStatement, PgFunc, Query, + InsertStatement, LockType, Order, PgFunc, Query, WithClause, }; use sea_orm::ActiveValue::NotSet; -use sea_orm::{prelude::*, FromQueryResult, Set, TransactionTrait}; -use std::collections::HashSet; +use sea_orm::{prelude::*, ConnectionTrait, FromQueryResult, Set, TransactionTrait}; +use std::collections::{HashMap, HashSet}; use uuid::Uuid; use crate::entity::role::{GroupWorkerRole, UserGroupRole}; @@ -45,9 +45,15 @@ pub(crate) fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { Ok(()) } +#[derive(Clone, Copy)] enum Submitter { User, - Worker(Uuid), + /// A running task spawning a downstream child, identified by the parent's + /// uuid and the group the parent itself lives in. + Task { + upstream_task_uuid: Uuid, + parent_group_id: i64, + }, } async fn internal_submit_task( @@ -71,7 +77,9 @@ async fn internal_submit_task( // Used later when creating active tasks active model let (state, upstream_task_uuid) = match submitter { Submitter::User => (Set(crate::entity::state::TaskState::Ready), NotSet), - Submitter::Worker(upstream_task_uuid) => ( + Submitter::Task { + upstream_task_uuid, .. + } => ( Set(crate::entity::state::TaskState::Pending), Set(Some(upstream_task_uuid)), ), @@ -89,54 +97,18 @@ async fn internal_submit_task( .transaction::<_, (ActiveTasks::Model, Option), crate::error::Error>( |txn| { Box::pin(async move { - // Determine the owning group and (optionally) the suite, then - // atomically bump the group's task counter. When a suite is designated it - // is authoritative: the task's group is the suite's owning group, and the - // request's group_name must match it. - let (group_id, task_id, suite) = match suite_uuid { - None => { - // No suite: resolve the user-specified group, verify the caller's - // membership, and bump its task counter in one statement. - // No row back means either the group doesn't exist or the - // caller isn't in it; both return 404 so users won't know - // about the existence of a group they are not a member of. - // Membership must stay a subquery, not a join: both table has a - // `id` column and sea-orm will create a unqualified `RETURNING id`, - // which Postgres rejects as ambiguous. - let member_of = Query::select() - .column(UserGroup::Column::GroupId) - .from(UserGroup::Entity) - .and_where(Expr::col(UserGroup::Column::UserId).eq(creator_id)) - // TODO: when there is 'exec' access level, enforce access check here. - // - // .and_where( - // Expr::col(UserGroup::Column::Role) - // .gte(UserGroupRole::Write), - // ) - .to_owned(); - let group = Group::Entity::update_many() - .col_expr( - Group::Column::TaskCount, - Expr::col((Group::Entity, Group::Column::TaskCount)).add(1), - ) - .col_expr(Group::Column::UpdatedAt, Expr::value(now)) - .filter(Group::Column::GroupName.eq(&group_name)) - .filter(Group::Column::Id.in_subquery(member_of)) - .exec_with_returning(txn) - .await? - .into_iter() - .next() - .ok_or_else(|| { - Error::ApiError(crate::error::ApiError::NotFound(format!( - "User doesn't have permission or group with name {}", - group_name - ))) - })?; - (group.id, group.task_count, None) - } - Some(suite_uuid) => { - // Suite designated: resolve it (and the caller's membership in its - // owning group) in one join. The suite is authoritative for the group. + // Resolve the designated suite, if any. How it is authorized + // depends on the submitter: a user reaches a suite through their + // own group memberships, while a task spawning a child may target + // any suite owned by the group the parent task itself lives in. + // + // The parent's creator is deliberately not re-checked, so a long-running suite + // keeps spawning work after that user leaves the group. + let suite = match (suite_uuid, submitter) { + (None, _) => None, + (Some(suite_uuid), Submitter::User) => { + // Resolve the suite and the caller's membership in its + // owning group in one join. let builder = txn.get_database_backend(); let suite_stmt = Query::select() .columns([ @@ -189,64 +161,195 @@ async fn internal_submit_task( suite_uuid ))) })?; - // The suite must be able to accept new tasks. - if !suite.state.can_accept_tasks() { - return Err(Error::ApiError( - crate::error::ApiError::InvalidRequest(format!( - "Suite is in {} state and cannot accept new tasks", - suite.state - )), - )); - } - // Derive the owning group from the suite and bump its counter. + Some(suite) + } + ( + Some(suite_uuid), + Submitter::Task { + parent_group_id, .. + }, + ) => Some( + TaskSuites::Entity::find() + .filter(TaskSuites::Column::Uuid.eq(suite_uuid)) + .filter(TaskSuites::Column::GroupId.eq(parent_group_id)) + .one(txn) + .await? + .ok_or_else(|| { + // A suite owned by another group is reported the + // same way as one that does not exist. + Error::ApiError(crate::error::ApiError::NotFound(format!( + "Suite with uuid {} in the parent task's group", + suite_uuid + ))) + })?, + ), + }; + + // A first, unlocked look, so a suite that is plainly closed to + // new work is rejected with its actual state named. The + // binding check is the `WHERE` on the counter bump below — + // this one is only the better error message. + if let Some(suite) = &suite { + if !suite.state.can_accept_tasks() { + return Err(Error::ApiError(crate::error::ApiError::InvalidRequest( + format!( + "Suite is in {} state and cannot accept new tasks", + suite.state + ), + ))); + } + } + + // The owning group, when it is already pinned by something other + // than `group_name`: a designated suite owns the task, and a + // task-spawned child always stays in its parent's group. `None` + // is the plain user path, where the group is resolved from + // `group_name` and the caller's membership. + let pinned_group_id = match (&suite, submitter) { + (Some(suite), _) => Some(suite.group_id), + ( + None, + Submitter::Task { + parent_group_id, .. + }, + ) => Some(parent_group_id), + (None, Submitter::User) => None, + }; + + // Bump the owning group's task counter atomically. + let (group_id, task_id) = match pinned_group_id { + Some(pinned_group_id) => { let group = Group::Entity::update_many() .col_expr( Group::Column::TaskCount, Expr::col(Group::Column::TaskCount).add(1), ) .col_expr(Group::Column::UpdatedAt, Expr::value(now)) - .filter(Group::Column::Id.eq(suite.group_id)) + .filter(Group::Column::Id.eq(pinned_group_id)) .exec_with_returning(txn) .await? .into_iter() .next() .ok_or_else(|| { - // The suite's group row must exist (FK) - tracing::error!( - "Owning group {} of suite {} not found", - suite.group_id, - suite_uuid - ); + // The row must exist: both the suite and the + // parent task hold an FK to it. + tracing::error!("Owning group {} not found", pinned_group_id); Error::ApiError(crate::error::ApiError::InternalServerError) })?; - // The request's group_name must match the suite's owning group. - // Safe to reveal the real group: access to the suite is proven. + // The request's group_name must match if group.group_name != group_name { + let owner = match &suite { + Some(suite) => { + format!( + "Suite {} belongs to group {}", + suite.uuid, group.group_name + ) + } + None => format!( + "The parent task belongs to group {}", + group.group_name + ), + }; return Err(Error::ApiError( crate::error::ApiError::InvalidRequest(format!( - "Suite {} belongs to group {}, not {}", - suite_uuid, group.group_name, group_name + "{}, not {}", + owner, group_name )), )); } - (group.id, group.task_count, Some(suite)) + (group.id, group.task_count) + } + None => { + // Resolve the user-specified group, verify the caller's + // membership, and bump its task counter in one statement. + // No row back means either the group doesn't exist or the + // caller isn't in it; both return 404 so users won't know + // about the existence of a group they are not a member of. + // Membership must stay a subquery, not a join: both table has a + // `id` column and sea-orm will create a unqualified `RETURNING id`, + // which Postgres rejects as ambiguous. + let member_of = Query::select() + .column(UserGroup::Column::GroupId) + .from(UserGroup::Entity) + .and_where(Expr::col(UserGroup::Column::UserId).eq(creator_id)) + // TODO: when there is 'exec' access level, enforce access check here. + // + // .and_where( + // Expr::col(UserGroup::Column::Role) + // .gte(UserGroupRole::Write), + // ) + .to_owned(); + let group = Group::Entity::update_many() + .col_expr( + Group::Column::TaskCount, + Expr::col((Group::Entity, Group::Column::TaskCount)).add(1), + ) + .col_expr(Group::Column::UpdatedAt, Expr::value(now)) + .filter(Group::Column::GroupName.eq(&group_name)) + .filter(Group::Column::Id.in_subquery(member_of)) + .exec_with_returning(txn) + .await? + .into_iter() + .next() + .ok_or_else(|| { + Error::ApiError(crate::error::ApiError::NotFound(format!( + "User doesn't have permission or group with name {}", + group_name + ))) + })?; + (group.id, group.task_count) } }; + // One statement, so both increments are evaluated server-side + // on the row this `UPDATE` itself locked. Read the counters + // first and write literals back and two concurrent submits + // both see `total = n` and both write `n + 1`: the suite + // undercounts, `decrement_incomplete_tasks` reaches zero with + // tasks still `Ready`, and the surplus is stranded in a + // `Complete` suite that hands out nothing. + // + // The state predicate carries the accepts-tasks check, so no + // row is bumped after a cancellation that committed since the + // read above. let suite = match suite { None => None, - Some(suite) => { - let prev_total_tasks = suite.total_tasks; - let prev_incomplete_tasks = suite.incomplete_tasks; - let mut suite: TaskSuites::ActiveModel = suite.into(); - suite.total_tasks = Set(prev_total_tasks + 1); - suite.incomplete_tasks = Set(prev_incomplete_tasks + 1); - suite.last_task_submitted_at = Set(Some(now)); - suite.updated_at = Set(now); - suite.state = Set(TaskSuiteState::Open); - suite.completed_at = Set(None); - Some(suite.update(txn).await?) - } + Some(suite) => Some( + TaskSuites::Entity::update_many() + .col_expr( + TaskSuites::Column::TotalTasks, + Expr::col(TaskSuites::Column::TotalTasks).add(1), + ) + .col_expr( + TaskSuites::Column::IncompleteTasks, + Expr::col(TaskSuites::Column::IncompleteTasks).add(1), + ) + .col_expr( + TaskSuites::Column::LastTaskSubmittedAt, + Expr::value(Some(now)), + ) + .col_expr(TaskSuites::Column::UpdatedAt, Expr::value(now)) + .col_expr( + TaskSuites::Column::State, + Expr::value(TaskSuiteState::Open), + ) + .col_expr( + TaskSuites::Column::CompletedAt, + Expr::value(None::), + ) + .filter(TaskSuites::Column::Id.eq(suite.id)) + .filter(TaskSuites::Column::State.ne(TaskSuiteState::Cancelled)) + .exec_with_returning(txn) + .await? + .into_iter() + .next() + .ok_or_else(|| { + // Cancelled between the read above and here. + Error::ApiError(crate::error::ApiError::InvalidRequest( + format!("Suite {} stopped accepting new tasks", suite.uuid), + )) + })?, + ), }; let task_uuid = Uuid::new_v4(); @@ -267,9 +370,24 @@ async fn internal_submit_task( exec_options: Set(exec_options_json), result: Set(None), upstream_task_uuid, + task_suite_id: Set(suite.as_ref().map(|s| s.id)), ..Default::default() }; let task = task.insert(txn).await?; + + // Queued here, inside the transaction, and never after it + // commits. The queue is emptied by `served`, which removes + // what a claim locked — so a task pushed before its commit + // is always pushed before any `served` that could take it + // back out, because no claim can lock a row it cannot yet + // see. Push after the commit instead and a claim landing in + // between wins the race: it takes the task, `served` finds + // nothing in the queue to remove, and the push then puts an + // already-running task back on the suite's backlog, where + // nothing will ever remove it again. + // TODO: once suites have a ready queue, push a `Ready` suite task + // onto it here with `queues.push_ready(suite.id, [(task.id, task.priority)])`. + Ok((task, suite)) }) }, @@ -339,13 +457,29 @@ pub async fn user_submit_task( internal_submit_task(pool, creator_id, Submitter::User, req).await } +/// Submit the downstream child of a running task, on behalf of the worker or +/// agent executing it. +/// +/// The child stays in the parent's group, and `req.suite_uuid` may name any +/// suite that group owns — the parent's own suite, a sibling one, or `None` for +/// a suite-less task dispatched to workers. pub async fn worker_submit_pending_task( pool: &InfraPool, creator_id: i64, upstream_task_uuid: Uuid, + parent_group_id: i64, req: SubmitTaskReq, ) -> crate::error::Result { - internal_submit_task(pool, creator_id, Submitter::Worker(upstream_task_uuid), req).await + internal_submit_task( + pool, + creator_id, + Submitter::Task { + upstream_task_uuid, + parent_group_id, + }, + req, + ) + .await } pub async fn worker_trigger_pending_task(pool: &InfraPool, uuid: Uuid) -> crate::error::Result<()> { @@ -362,6 +496,13 @@ pub async fn worker_trigger_pending_task(pool: &InfraPool, uuid: Uuid) -> crate: task.state = Set(TaskState::Ready); task.updated_at = Set(TimeDateTimeWithTimeZone::now_utc()); let task = task.update(&pool.db).await?; + // A suite task never enters the worker queues — agents pull it from its + // suite. Wake the idle eligible agents instead. + if let Some(_suite_id) = task.task_suite_id { + // TODO: once agents can be notified, call + // `notify_suite_available(pool, _suite_id)` here. + return Ok(()); + } // Batch add task to worker task queues let builder = pool.db.get_database_backend(); let tasks_stmt = Query::select() @@ -613,6 +754,7 @@ pub async fn user_cancel_task( } } let now = TimeDateTimeWithTimeZone::now_utc(); + let suite_id = task.task_suite_id; let res = TaskResultSpec { exit_status: 0, msg: Some(crate::schema::TaskResultMessage::UserCancellation), @@ -638,6 +780,14 @@ pub async fn user_cancel_task( downstream_task_uuid: Set(task.downstream_task_uuid), task_suite_id: Set(task.task_suite_id), }; + // A cancelled suite task never reaches an agent `Commit`, so + // this is the only chance to give the suite its count back. It + // goes first so the suite row is taken before the task rows — + // the order every writer of the pair uses. + if let Some(suite_id) = suite_id { + crate::service::suite::decrement_incomplete_tasks(txn, suite_id, 1, now) + .await?; + } archived_task.insert(txn).await?; ActiveTasks::Entity::delete_by_id(task.id).exec(txn).await?; Ok(task.id) @@ -646,6 +796,8 @@ pub async fn user_cancel_task( .await?; let _ = remove_task(task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); + // TODO: once suites have a ready queue, remove a cancelled suite task from it + // as well. Ok(()) } @@ -1385,6 +1537,34 @@ pub async fn query_tasks_by_filter( #[derive(FromQueryResult)] struct IdResult { id: i64, + task_suite_id: Option, +} + +/// Tick `incomplete_tasks` down once per cancelled task, for every suite a +/// batch cancel touched. +/// +/// The rows come straight from the archive's `RETURNING`, so this counts what +/// was actually cancelled rather than what was asked for. +async fn decrement_incomplete_tasks_by_suite( + txn: &C, + suite_ids: impl IntoIterator>, + now: TimeDateTimeWithTimeZone, +) -> crate::error::Result<()> { + let mut per_suite: HashMap = HashMap::new(); + for suite_id in suite_ids.into_iter().flatten() { + *per_suite.entry(suite_id).or_default() += 1; + } + // Sorted, not iterated straight off the map: each decrement writes its + // suite row and holds that lock to commit, so two cancels over overlapping + // suites in opposite orders deadlock. The `locked` CTE of each batch cancel + // has normally taken these rows already, which makes this a backstop — it + // still matters for the straggler suite that CTE cannot cover. + let mut per_suite: Vec<(i64, i32)> = per_suite.into_iter().collect(); + per_suite.sort_unstable_by_key(|(suite_id, _)| *suite_id); + for (suite_id, count) in per_suite { + crate::service::suite::decrement_incomplete_tasks(txn, suite_id, count, now).await?; + } + Ok(()) } /// Cancel multiple tasks by filter criteria. @@ -1464,17 +1644,71 @@ pub async fn cancel_tasks_by_filter( // Build the complete CTE statement before the transaction to avoid lifetime issues + // The suite rows this cancel will decrement. Every other writer of the pair + // takes the suite row before the task rows, and this is how a batch cancel + // joins them — on its own it learns its suites from the archive's `RETURNING`, + // far too late to lock anything. + // + // In the same statement as the delete, so one snapshot covers both: the filter + // selects the same tasks in each CTE, and a task submitted in between is + // invisible to either. + // + // `ORDER BY id` is load-bearing. Postgres puts `LockRows` above the sort, so + // rows are locked in id order; any scan is therefore a prefix of ascending + // ids, and overlapping cancels cannot take two rows in opposite orders. + let locked_suites = Query::select() + .column(TaskSuites::Column::Id) + .from(TaskSuites::Entity) + .and_where( + Expr::col(TaskSuites::Column::Id).in_subquery( + Query::select() + .column(ActiveTasks::Column::TaskSuiteId) + .from(ActiveTasks::Entity) + .and_where( + Expr::col(ActiveTasks::Column::Id).in_subquery(task_id_subquery.clone()), + ) + .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).is_not_null()) + .to_owned(), + ), + ) + .order_by(TaskSuites::Column::Id, Order::Asc) + .lock(LockType::Update) + .to_owned(); + let locked_cte = CommonTableExpression::new() + .query(locked_suites) + .table_name(Alias::new("locked")) + .to_owned(); + let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::Id).in_subquery(task_id_subquery)) + // This predicate is the guarantee, not a nicety: a task is deleted only if + // its suite came out of `locked`, and producing that row is what locked it. + // So the suite row is always taken before the task row, whatever plan the + // subquery gets. + .and_where( + Expr::col(ActiveTasks::Column::TaskSuiteId) + .is_null() + .or(Expr::col(ActiveTasks::Column::TaskSuiteId).in_subquery( + Query::select() + .expr(Expr::col(Alias::new("id"))) + .from(Alias::new("locked")) + .to_owned(), + )), + ) .returning_all() .to_owned(); - let cte = CommonTableExpression::new() + let deleted_cte = CommonTableExpression::new() .query(delete_stmt) .table_name(Alias::new("deleted")) .to_owned(); + let with_ctes = WithClause::new() + .cte(locked_cte) + .cte(deleted_cte) + .to_owned(); + // Get the database backend before the transaction let builder = pool.db.get_database_backend(); @@ -1483,7 +1717,7 @@ pub async fn cancel_tasks_by_filter( let task_ids = pool .db .transaction::<_, Vec, crate::error::Error>(|txn| { - let cte = cte.clone(); + let with_ctes = with_ctes.clone(); Box::pin(async move { let now = TimeDateTimeWithTimeZone::now_utc(); let res = TaskResultSpec { @@ -1511,6 +1745,7 @@ pub async fn cancel_tasks_by_filter( .expr(Expr::value(result.clone())) .expr(Expr::col(Alias::new("upstream_task_uuid"))) .expr(Expr::col(Alias::new("downstream_task_uuid"))) + .expr(Expr::col(Alias::new("task_suite_id"))) .from(Alias::new("deleted")) .to_owned(); @@ -1535,21 +1770,28 @@ pub async fn cancel_tasks_by_filter( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, + // Without this the archived copy of a suite task loses + // its suite, and the count below has nothing to key on. + ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); insert_stmt.select_from(select_from_cte).unwrap(); - insert_stmt.returning_col(ArchivedTasks::Column::Id); - let insert_with_cte = insert_stmt.with(cte.into()); + insert_stmt.returning_all(); + let insert_with_cte = insert_stmt.with(with_ctes); let stmt = builder.build(&insert_with_cte); - let task_ids: Vec = IdResult::find_by_statement(stmt) - .all(txn) - .await? - .into_iter() - .map(|r| r.id) - .collect(); + let cancelled: Vec = IdResult::find_by_statement(stmt).all(txn).await?; + + decrement_incomplete_tasks_by_suite( + txn, + cancelled.iter().map(|r| r.task_suite_id), + now, + ) + .await?; + + let task_ids: Vec = cancelled.into_iter().map(|r| r.id).collect(); Ok(task_ids) }) @@ -1561,6 +1803,8 @@ pub async fn cancel_tasks_by_filter( let _ = remove_task(*task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); } + // TODO: once suites have a ready queue, remove the cancelled suite tasks + // from it as well. Ok(TasksCancelByFilterResp { cancelled_count: task_ids.len() as u64, @@ -1572,6 +1816,7 @@ pub async fn cancel_tasks_by_filter( struct IdUuidResult { id: i64, uuid: Uuid, + task_suite_id: Option, } pub async fn cancel_tasks_by_uuids( @@ -1635,23 +1880,77 @@ pub async fn cancel_tasks_by_uuids( // Build CTE for DELETE RETURNING + INSERT SELECT to avoid parameter limits // Convert the SELECT statement into a subquery for the DELETE + // The suite rows this cancel will decrement. Every other writer of the pair + // takes the suite row before the task rows, and this is how a batch cancel + // joins them — on its own it learns its suites from the archive's `RETURNING`, + // far too late to lock anything. + // + // In the same statement as the delete, so one snapshot covers both: the filter + // selects the same tasks in each CTE, and a task submitted in between is + // invisible to either. + // + // `ORDER BY id` is load-bearing. Postgres puts `LockRows` above the sort, so + // rows are locked in id order; any scan is therefore a prefix of ascending + // ids, and overlapping cancels cannot take two rows in opposite orders. + let locked_suites = Query::select() + .column(TaskSuites::Column::Id) + .from(TaskSuites::Entity) + .and_where( + Expr::col(TaskSuites::Column::Id).in_subquery( + Query::select() + .column(ActiveTasks::Column::TaskSuiteId) + .from(ActiveTasks::Entity) + .and_where( + Expr::col(ActiveTasks::Column::Id).in_subquery(id_subquery.clone()), + ) + .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).is_not_null()) + .to_owned(), + ), + ) + .order_by(TaskSuites::Column::Id, Order::Asc) + .lock(LockType::Update) + .to_owned(); + let locked_cte = CommonTableExpression::new() + .query(locked_suites) + .table_name(Alias::new("locked")) + .to_owned(); + let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::Id).in_subquery(id_subquery)) + // This predicate is the guarantee, not a nicety: a task is deleted only if + // its suite came out of `locked`, and producing that row is what locked it. + // So the suite row is always taken before the task row, whatever plan the + // subquery gets. + .and_where( + Expr::col(ActiveTasks::Column::TaskSuiteId) + .is_null() + .or(Expr::col(ActiveTasks::Column::TaskSuiteId).in_subquery( + Query::select() + .expr(Expr::col(Alias::new("id"))) + .from(Alias::new("locked")) + .to_owned(), + )), + ) .returning_all() .to_owned(); - let cte = CommonTableExpression::new() + let deleted_cte = CommonTableExpression::new() .query(delete_stmt) .table_name(Alias::new("deleted")) .to_owned(); + let with_ctes = WithClause::new() + .cte(locked_cte) + .cte(deleted_cte) + .to_owned(); + // Execute delete and insert in a single transaction // Total: 1 query using CTE (DELETE RETURNING + INSERT SELECT) per chunk let (task_ids, found_uuids) = pool .db .transaction::<_, (Vec, HashSet), crate::error::Error>(|txn| { - let cte = cte.clone(); + let with_ctes = with_ctes.clone(); Box::pin(async move { let now = TimeDateTimeWithTimeZone::now_utc(); let res = TaskResultSpec { @@ -1680,6 +1979,7 @@ pub async fn cancel_tasks_by_uuids( .expr(Expr::value(result.clone())) .expr(Expr::col(Alias::new("upstream_task_uuid"))) .expr(Expr::col(Alias::new("downstream_task_uuid"))) + .expr(Expr::col(Alias::new("task_suite_id"))) .from(Alias::new("deleted")) .to_owned(); @@ -1704,18 +2004,29 @@ pub async fn cancel_tasks_by_uuids( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, + // Without this the archived copy of a suite task + // loses its suite, and the count below has nothing + // to key on. + ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); insert_stmt.select_from(select_from_cte).unwrap(); insert_stmt.returning_all(); - let insert_with_cte = insert_stmt.with(cte.into()); + let insert_with_cte = insert_stmt.with(with_ctes); let stmt = builder.build(&insert_with_cte); let results: Vec = IdUuidResult::find_by_statement(stmt).all(txn).await?; + decrement_incomplete_tasks_by_suite( + txn, + results.iter().map(|r| r.task_suite_id), + now, + ) + .await?; + let task_ids: Vec = results.iter().map(|r| r.id).collect(); let found_uuids: HashSet = results.into_iter().map(|r| r.uuid).collect(); @@ -1734,6 +2045,8 @@ pub async fn cancel_tasks_by_uuids( let _ = remove_task(*task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); } + // TODO: once suites have a ready queue, remove the cancelled suite tasks + // from it as well. // Determine which UUIDs failed (not found or no permission) let failed_uuids: Vec = req diff --git a/netmito/src/service/worker/mod.rs b/netmito/src/service/worker/mod.rs index 48298aa..6ea1f59 100644 --- a/netmito/src/service/worker/mod.rs +++ b/netmito/src/service/worker/mod.rs @@ -642,19 +642,9 @@ pub async fn report_task( if user.state != UserState::Active { return Err(AuthError::PermissionDenied.into()); } - // Load group information to verify group name matches - let group = Group::Entity::find_by_id(task.group_id) - .one(&pool.db) - .await? - .ok_or(ApiError::NotFound("Group not found".to_string()))?; - if req.group_name != group.group_name { - return Err(ApiError::InvalidRequest(format!( - "Group name mismatch: expected '{}', got '{}'", - group.group_name, req.group_name - )) - .into()); - } - // Verify worker has permission to access this group + // Verify worker has permission to access this group. The child's own + // group and suite are pinned to the parent's by the submit path, + // which also rejects a mismatched `req.group_name`. GroupWorker::Entity::find() .filter(GroupWorker::Column::WorkerId.eq(worker_id)) .filter(GroupWorker::Column::GroupId.eq(task.group_id)) @@ -664,17 +654,22 @@ pub async fn report_task( .ok_or(ApiError::AuthError( crate::error::AuthError::PermissionDenied, ))?; - let resp = - service::task::worker_submit_pending_task(pool, task.creator_id, task.uuid, req) - .await - .map_err(|e| match e { - crate::error::Error::AuthError(err) => ApiError::AuthError(err), - crate::error::Error::ApiError(e) => e, - _ => { - tracing::error!("{}", e); - ApiError::InternalServerError - } - })?; + let resp = service::task::worker_submit_pending_task( + pool, + task.creator_id, + task.uuid, + task.group_id, + req, + ) + .await + .map_err(|e| match e { + crate::error::Error::AuthError(err) => ApiError::AuthError(err), + crate::error::Error::ApiError(e) => e, + _ => { + tracing::error!("{}", e); + ApiError::InternalServerError + } + })?; let task = ActiveTask::ActiveModel { id: Set(task_id), updated_at: Set(now), @@ -718,10 +713,21 @@ pub async fn report_task( assigned_task_id: Set(None), ..Default::default() }; + let suite_id = task.task_suite_id; + // The suite's counter goes down in the same transaction that + // archives the task. Split across two, a failure in between leaves + // `incomplete_tasks` permanently too high, and a suite that never + // reaches zero never completes. It goes down *first* so this takes + // the suite row before the task rows, the order every writer of the + // pair uses. let (worker_id, worker_state) = pool .db - .transaction(|txn| { + .transaction::<_, (i64, WorkerState), Error>(|txn| { Box::pin(async move { + if let Some(suite_id) = suite_id { + service::suite::decrement_incomplete_tasks(txn, suite_id, 1, now) + .await?; + } archived_task.insert(txn).await?; ActiveTask::Entity::delete_by_id(task_id).exec(txn).await?; let worker = worker.update(txn).await?; From 9d62c5075498e852774b591d7ec2a327a17c8c6a Mon Sep 17 00:00:00 2001 From: un-lock-able Date: Tue, 27 Oct 2026 22:52:46 +0800 Subject: [PATCH 4/4] docs: rewrite suite and task comments for review --- netmito/src/client/mod.rs | 6 +- netmito/src/config/client/artifacts.rs | 8 +- netmito/src/config/client/mod.rs | 6 +- netmito/src/config/client/suites.rs | 71 +++---- netmito/src/config/client/tasks.rs | 8 +- netmito/src/schema/suite.rs | 44 ++-- netmito/src/schema/task.rs | 9 +- netmito/src/service/mod.rs | 30 +-- netmito/src/service/suite.rs | 212 ++++++++++--------- netmito/src/service/task.rs | 272 ++++++++++++------------- netmito/src/service/worker/mod.rs | 16 +- openapi.yaml | 87 ++++---- 12 files changed, 374 insertions(+), 395 deletions(-) diff --git a/netmito/src/client/mod.rs b/netmito/src/client/mod.rs index 530198a..98ccd76 100644 --- a/netmito/src/client/mod.rs +++ b/netmito/src/client/mod.rs @@ -1885,10 +1885,8 @@ impl MitoClient { output_task_info(&task); } } else { - // Bare uuids, except that a suite task says - // so: without it the terse listing is the - // one place a suite task is - // indistinguishable from a loose one. + // Without --verbose, the output shows only the task UUID. + // For a task in a suite, it also shows the suite UUID. for task in resp.tasks { match task.suite_uuid { Some(suite_uuid) => { diff --git a/netmito/src/config/client/artifacts.rs b/netmito/src/config/client/artifacts.rs index be5db64..bf5600d 100644 --- a/netmito/src/config/client/artifacts.rs +++ b/netmito/src/config/client/artifacts.rs @@ -88,8 +88,8 @@ pub struct DownloadArtifactsByFilterArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, - /// The UUID of the task suite the tasks belong to. Names the group on its - /// own, so `--group` is only needed to double-check it + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite #[arg(long = "suite")] pub suite_uuid: Option, /// The tags of the tasks @@ -199,8 +199,8 @@ pub struct DeleteArtifactsByFilterArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, - /// The UUID of the task suite the tasks belong to. Names the group on its - /// own, so `--group` is only needed to double-check it + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite #[arg(long = "suite")] pub suite_uuid: Option, /// The tags of the tasks diff --git a/netmito/src/config/client/mod.rs b/netmito/src/config/client/mod.rs index 96e12c0..50485fc 100644 --- a/netmito/src/config/client/mod.rs +++ b/netmito/src/config/client/mod.rs @@ -158,9 +158,9 @@ fn parse_artifact_content_type( } } -/// Parse a hook's `ExecSpec` from a JSON object, the same shape `POST /suites` -/// takes: `{"args":["sh","-c","./setup.sh"],"terminal_output":true}`. Only -/// `args` is required. +/// Parse a hook `ExecSpec` from a JSON object. The object has the same shape that `POST /suites` +/// accepts, for example `{"args":["sh","-c","./setup.sh"],"terminal_output":true}`. +/// Only `args` is necessary. fn parse_exec_spec( s: &str, ) -> Result> { diff --git a/netmito/src/config/client/suites.rs b/netmito/src/config/client/suites.rs index ae15fa9..cd3a08c 100644 --- a/netmito/src/config/client/suites.rs +++ b/netmito/src/config/client/suites.rs @@ -23,9 +23,10 @@ pub struct SuitesArgs { #[derive(Subcommand, Serialize, Debug, Deserialize, Clone)] pub enum SuitesCommands { /// Create a new task suite - // Boxed: the three optional hook specs make this variant far larger than the rest. + // Put the arguments in a box: the three optional hook specs make this variant much larger + // than the other variants. Create(Box), - /// Change a suite's properties. Takes effect on the jobs started after it + /// Change the properties of a suite. The change applies to the jobs that start after it Change(Box), /// Query task suites subject to a filter Query(QuerySuitesArgs), @@ -65,22 +66,24 @@ pub struct CreateSuiteArgs { /// Number of tasks each worker prefetches locally #[arg(long, default_value_t = 16)] pub prefetch: u32, - /// Provision hook, as a JSON exec spec: '{"args":["sh","-c","./setup.sh"],"terminal_output":true}'. - /// Runs once before any task; a non-zero exit fails the job and its tasks never start + /// The provision hook, as a JSON exec spec, for example + /// '{"args":["sh","-c","./setup.sh"],"terminal_output":true}'. It runs one time before the + /// tasks. If it exits with a non-zero code, the job fails and its tasks do not start #[arg(long, value_parser = parse_exec_spec)] pub provision: Option, - /// Cleanup hook, same JSON shape. Runs once after the tasks drain, even when the job already failed + /// The cleanup hook, with the same JSON shape. It runs one time after the last task of the + /// job ends. It also runs if the job failed #[arg(long, value_parser = parse_exec_spec)] pub cleanup: Option, - /// Background hook, same JSON shape. Runs alongside the tasks and is expected to outlive them + /// The background hook, with the same JSON shape. It runs at the same time as the tasks, and + /// usually continues to run until the job ends #[arg(long, value_parser = parse_exec_spec)] pub background: Option, } impl From for CreateTaskSuiteReq { fn from(args: CreateSuiteArgs) -> Self { - // A suite with no hook at all keeps `exec_hooks` absent rather than - // carrying three nulls. + // If no hook is given, keep `exec_hooks` absent. Do not send an object with three nulls. let exec_hooks = match (args.provision, args.cleanup, args.background) { (None, None, None) => None, (provision, cleanup, background) => Some(ExecHooks { @@ -106,61 +109,61 @@ impl From for CreateTaskSuiteReq { } } -/// Every field is left alone unless named. Each nullable one has a `--clear-*` -/// partner that conflicts with its value flag, so setting and clearing at once -/// is a parse error rather than a rejected request. +/// A field does not change if you do not give its flag. Each nullable field has a `--clear-*` +/// flag that conflicts with its value flag. If you set and clear the same field, clap shows a +/// parse error, and the client does not send the request. #[derive(Serialize, Debug, Deserialize, Args, Clone)] pub struct ChangeSuiteArgs { /// The UUID of the suite pub uuid: Uuid, - /// New human-readable name for the suite + /// The new name of the suite #[arg(short, long, conflicts_with = "clear_name")] pub name: Option, - /// Remove the suite's name + /// Remove the name of the suite #[arg(long)] pub clear_name: bool, - /// New description for the suite + /// The new description of the suite #[arg(short, long, conflicts_with = "clear_description")] pub description: Option, - /// Remove the suite's description + /// Remove the description of the suite #[arg(long)] pub clear_description: bool, - /// Replacement tags for agent matching (e.g. gpu,linux) + /// The new tags that an agent must have to run the suite (for example, gpu,linux) #[arg(short, long, num_args = 1.., value_delimiter = ',', conflicts_with = "clear_tags")] pub tags: Option>, - /// Drop every tag. An untagged suite matches every agent its group can write to + /// Remove all tags. A suite with no tags matches all agents that its group can write to #[arg(long)] pub clear_tags: bool, - /// Replacement labels for querying/filtering + /// The new labels, which you can use to query and filter suites #[arg(short, long, num_args = 1.., value_delimiter = ',', conflicts_with = "clear_labels")] pub labels: Option>, - /// Drop every label + /// Remove all labels #[arg(long)] pub clear_labels: bool, - /// Suite scheduling priority (higher = more important) + /// The scheduling priority of the suite. A higher value is more important #[arg(short, long)] pub priority: Option, - /// Number of workers each agent spawns for this suite. Must specify prefetch - /// setting alongside + /// The number of workers that each agent starts for this suite. You must also give + /// `--prefetch` #[arg(short, long, requires = "prefetch")] pub workers: Option, - /// Number of tasks each worker prefetches locally. Must specify worker count - /// settings alongside + /// The number of tasks that each worker gets before it needs them. You must also give + /// `--workers` #[arg(long, requires = "workers")] pub prefetch: Option, - /// Replacement provision hook, as a JSON exec spec + /// The new provision hook, as a JSON exec spec #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_provision")] pub provision: Option, /// Remove the provision hook #[arg(long)] pub clear_provision: bool, - /// Replacement cleanup hook, same JSON shape + /// The new cleanup hook, with the same JSON shape #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_cleanup")] pub cleanup: Option, /// Remove the cleanup hook #[arg(long)] pub clear_cleanup: bool, - /// Replacement background hook, same JSON shape + /// The new background hook, with the same JSON shape #[arg(long, value_parser = parse_exec_spec, conflicts_with = "clear_background")] pub background: Option, /// Remove the background hook @@ -168,8 +171,8 @@ pub struct ChangeSuiteArgs { pub clear_background: bool, } -/// A value flag and its `--clear-*` partner, as the one op they describe. -/// clap rules out the both-given case, so a value wins if one ever arrives. +/// Convert a value flag and its `--clear-*` flag into one `UpdateOp`. clap does not accept the +/// two flags together. If the two flags arrive, the value is used. fn update_op(value: Option, clear: bool) -> Option> { match (value, clear) { (Some(value), _) => Some(UpdateOp::SetTo(value)), @@ -180,8 +183,8 @@ fn update_op(value: Option, clear: bool) -> Option> { impl From for ChangeTaskSuiteReq { fn from(args: ChangeSuiteArgs) -> Self { - // An array column has no null, so its cleared state is the empty set — - // the flag pair is uniform at the CLI even where the request is not. + // An array column cannot be null, so its cleared value is an empty set. The CLI uses the + // same flag pair for all fields, but the request is different for these two fields. let tags = match (args.tags, args.clear_tags) { (Some(tags), _) => Some(tags.into_iter().collect()), (None, true) => Some(HashSet::new()), @@ -192,9 +195,9 @@ impl From for ChangeTaskSuiteReq { (None, true) => Some(HashSet::new()), (None, false) => None, }; - // The plan is replaced wholesale, so it is stated in full or not at all: - // clap requires the two flags together, and neither half may default over - // what the suite already had. + // The request replaces the full plan, so it must contain the full plan or no plan. clap + // requires the two flags together. A missing flag must not get a default value that + // replaces the current value of the suite. let worker_schedule = match (args.workers, args.prefetch) { (Some(worker_count), Some(prefetch)) => Some(WorkerSchedulePlan::FixedWorkers { worker_count, diff --git a/netmito/src/config/client/tasks.rs b/netmito/src/config/client/tasks.rs index 7a97587..7023ebe 100644 --- a/netmito/src/config/client/tasks.rs +++ b/netmito/src/config/client/tasks.rs @@ -96,8 +96,8 @@ pub struct QueryTasksArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, - /// The UUID of the task suite the tasks belong to. Names the group on its - /// own, so `--group` is only needed to double-check it + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite #[arg(long = "suite")] pub suite_uuid: Option, /// The tags of the tasks @@ -146,8 +146,8 @@ pub struct CancelTasksArgs { /// The name of the group the tasks belong to #[arg(short, long)] pub group: Option, - /// The UUID of the task suite the tasks belong to. Names the group on its - /// own, so `--group` is only needed to double-check it + /// The UUID of the task suite that the tasks belong to. The suite also identifies the group. + /// If you also give `--group`, it must be the group of the suite #[arg(long = "suite")] pub suite_uuid: Option, /// The tags of the tasks diff --git a/netmito/src/schema/suite.rs b/netmito/src/schema/suite.rs index e400eeb..c43654c 100644 --- a/netmito/src/schema/suite.rs +++ b/netmito/src/schema/suite.rs @@ -43,12 +43,12 @@ pub struct CreateTaskSuiteResp { pub uuid: Uuid, } -/// A change to a field whose column is nullable. The field itself is wrapped in -/// an `Option`, where absent means "leave alone" — so the two variants are the -/// only things a caller can ask for, and neither can be confused with silence. +/// A change to a field that has a nullable column. The request puts the field in an `Option`, +/// and an absent field does not change. +/// A present field has one of these two values: set a new value, or set the column to null. /// -/// Fields backed by a `NOT NULL` column keep a plain `Option`: they have no null -/// to set, and their cleared state is a value (an empty array, say). +/// A field that has a `NOT NULL` column uses a plain `Option`. This column cannot be null, so its +/// 'cleared' value is a value of its type, for example an empty array. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "op", content = "value", rename_all = "snake_case")] pub enum UpdateOp { @@ -57,7 +57,7 @@ pub enum UpdateOp { } impl UpdateOp { - /// The value this op stores. + /// Returns the value to store. `SetNull` returns `None`. pub fn into_value(self) -> Option { match self { Self::SetTo(value) => Some(value), @@ -66,13 +66,13 @@ impl UpdateOp { } } -/// Per-hook changes to a suite's `exec_hooks`. Each hook is set, cleared, or — -/// absent — left as it was; the alternative, replacing the whole object, would -/// drop the hooks a caller merely failed to mention. Clearing all three stores -/// no hooks at all, the shape creation leaves behind for a suite with none. +/// The changes to the `exec_hooks` of a suite, one for each hook. You can set a hook, clear a +/// hook, or leave a hook out to keep it. The request does not replace the full object, because +/// that would remove each hook that the request does not mention. If you clear all three hooks, +/// the suite stores NULL, the same as a suite that you create without hooks. /// -/// The merge itself is done in SQL, over the column's own value, so two -/// concurrent patches cannot lose each other's edit. +/// The database merges the changes into the current value of the column. Because of this, two +/// changes at the same time cannot overwrite each other. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ExecHooksUpdate { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -89,28 +89,26 @@ impl ExecHooksUpdate { } } -/// Request to change a suite's properties. Every field is absent-means-unchanged, -/// and an empty request is rejected rather than treated as a no-op. +/// A request to change the properties of a suite. An absent field means no change. +/// The server rejects an empty request. /// -/// A change reaches the jobs started after it, never the ones already running: -/// slot counts and hooks are snapshotted when a job starts, and an agent that -/// stops matching the new tags still finishes what it holds. +/// The change applies to the jobs that start after it, because a job copies the worker schedule +/// and the hooks when it starts. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ChangeTaskSuiteReq { #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option>, - /// Tags for agent matching. An empty set matches every agent the suite's - /// group can write to, since matching is array containment. + /// The tags that an agent must have to run the suite. An empty set matches all agents that + /// the group of the suite can write to. #[serde(default, skip_serializing_if = "Option::is_none")] pub tags: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub labels: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub priority: Option, - /// Replaces the plan wholesale — a patch could not express switching to - /// another scheduling policy. + /// Replaces the full plan. A partial change cannot change the plan to a different scheduling policy. #[serde(default, skip_serializing_if = "Option::is_none")] pub worker_schedule: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -128,8 +126,8 @@ impl ChangeTaskSuiteReq { && self.exec_hooks.as_ref().map_or(true, |h| h.is_empty()) } - /// Whether the change touches what the scheduler reads: who may run the - /// suite, and where it sits in the pick order. + /// Returns true if the change can change the result of the scheduler: which agents can run the + /// suite, or the position of the suite in the pick order. pub fn affects_scheduling(&self) -> bool { self.tags.is_some() || self.priority.is_some() } diff --git a/netmito/src/schema/task.rs b/netmito/src/schema/task.rs index 0abe9b4..89b29d3 100644 --- a/netmito/src/schema/task.rs +++ b/netmito/src/schema/task.rs @@ -100,8 +100,7 @@ pub struct TaskQueryInfo { pub uuid: Uuid, pub creator_username: String, pub group_name: String, - /// The suite that owns this task, if any. A suite always belongs to - /// `group_name`, since submitting to one pins the task's group to it. + /// The suite of this task, if the task is in a suite. The `group_name` above must own this suite. pub suite_uuid: Option, pub task_id: i64, pub tags: Vec, @@ -146,9 +145,9 @@ pub struct ParsedTaskQueryInfo { pub struct TasksQueryReq { pub creator_usernames: Option>, pub group_name: Option, - /// Restrict to the tasks of one suite. The suite names its own group, so it - /// fills in `group_name` when that was left out and contradicts it - /// otherwise. + /// Show only the tasks of this suite. The suite also identifies the group. If `group_name` + /// is absent, the server uses the group of the suite. If `group_name` is a different group, + /// the server returns 404, the same as for a suite that does not exist. pub suite_uuid: Option, pub tags: Option>, pub labels: Option>, diff --git a/netmito/src/service/mod.rs b/netmito/src/service/mod.rs index c3aabac..f3229fa 100644 --- a/netmito/src/service/mod.rs +++ b/netmito/src/service/mod.rs @@ -1,24 +1,24 @@ //! # Lock order: suite before task //! -//! A transaction that writes both a `task_suites` row and `active_tasks` rows -//! must write the suite row first. +//! A transaction that writes a `task_suites` row and `active_tasks` rows must write the suite +//! row first. //! -//! Row locks taken by an `UPDATE` or `DELETE` live until the transaction -//! commits, not until the statement ends — so single-statement writes do not -//! make ordering someone else's problem. A transaction accumulates its locks, -//! and two of them taking the same pair in opposite orders deadlock, after a -//! full `deadlock_timeout` of waiting first. +//! An `UPDATE` or `DELETE` keeps its row locks until the transaction commits, not until the +//! statement ends. Thus, a write in one statement does not solve the order problem. A +//! transaction keeps all of its locks. If two transactions lock the same two rows in opposite +//! order, they deadlock. Postgres finds the deadlock only after `deadlock_timeout`. //! -//! Two shapes satisfy it: +//! There are two correct ways to keep this order: //! -//! - The suite id is known up front, as on every commit and single-task -//! cancel: write the suite row first and let the rest follow. -//! - The suite ids are only derivable from the task rows, as on a batch -//! cancel: lock them in the same statement that removes the tasks, and gate -//! the removal on the lock, so no task is taken before its suite is. +//! - The suite id is known before the write, for example on each commit and on a single-task +//! cancel. Write the suite row first, then the task rows. +//! - The suite ids come only from the task rows, for example on a batch cancel. Lock the suites +//! in the same statement that removes the tasks, and remove a task only after its suite is +//! locked. //! -//! Touching task rows without writing a suite row is exempt: `claim_candidates` -//! holds `FOR UPDATE SKIP LOCKED` on tasks alone, and nothing waits on it. +//! This rule does not apply to a transaction that locks task rows and does not write a suite +//! row. `claim_candidates` locks only task rows with `FOR UPDATE SKIP LOCKED`, and no +//! transaction waits for these locks. pub mod auth; pub mod group; diff --git a/netmito/src/service/suite.rs b/netmito/src/service/suite.rs index 3b7aa94..8407de6 100644 --- a/netmito/src/service/suite.rs +++ b/netmito/src/service/suite.rs @@ -36,28 +36,27 @@ struct TaskUuidResult { uuid: Uuid, } -/// Hand `count` finished tasks back to a suite: tick `incomplete_tasks` down, -/// and complete the suite if that empties an already-`Closed` one. +/// Decrease `incomplete_tasks` of a suite by `count`, because `count` of its tasks are finished. +/// If this makes the counter of a `Closed` suite zero, the suite becomes `Complete`. /// -/// Every path that takes an active task out of a suite ends here. A path that -/// archives a suite task without calling this parks the -/// suite short of `Complete` and leaves agents cycling over work that is no -/// longer there. +/// Each path that removes an active task from a suite must call this function. If a path archives +/// a suite task and does not call it, the suite cannot become `Complete`. Agents then continue to +/// look for tasks that do not exist. /// -/// The subtraction is deliberately unguarded. Every caller has just removed -/// exactly `count` rows from `active_tasks`, and each of those rows added one to -/// the counter when it was submitted, so the arithmetic cannot legitimately go -/// below zero. Clamping it would only turn a bug elsewhere into a suite that -/// looks fine; a negative counter is reported instead, loudly. +/// The subtraction has no lower limit, and this is intentional. Each caller removed exactly `count` +/// rows from `active_tasks` immediately before the call. Each of these rows increased the counter +/// by one when it was submitted. Thus, the counter cannot correctly go below zero. A lower limit +/// would hide a bug in a different part of the code. Instead, the function logs an error for a +/// negative counter. /// -/// Draining an `Open` suite does not complete it: it stays `Open` until it has -/// also been quiet for `suite_auto_close_timeout`, which is what lets the agent -/// hold its job — and its provisioned environment — open for a late arrival. -/// [`sweep_inactive_suites`] settles it afterwards. A `Closed` suite has no such -/// window to wait out, so it is completed here. +/// When an `Open` suite has no more tasks, it does not become `Complete` here. It stays `Open` +/// until it also gets no new task for `suite_auto_close_timeout`. During this time, the agent can +/// keep its job and its provisioned environment for a task that arrives late. Then +/// [`sweep_inactive_suites`] changes the state of the suite. A `Closed` suite does not wait, so +/// this function makes it `Complete` immediately. /// -/// The suite row must be taken before any task row, either by this call or by -/// the caller before it — see the lock-order note on [`crate::service`]. +/// The suite row must be locked before all task rows, by this call or by the caller before it. +/// Refer to the lock-order note on [`crate::service`]. pub(crate) async fn decrement_incomplete_tasks( db: &C, suite_id: i64, @@ -78,8 +77,8 @@ pub(crate) async fn decrement_incomplete_tasks( .exec_with_returning(db) .await?; - // Nothing here can repair it, and swallowing it would leave a suite that - // never completes and never gets claimed, with no trace of why. + // This function cannot repair a negative counter. It does not set the counter to zero, so + // that the bug is easier to find. for suite in updated.iter().filter(|s| s.incomplete_tasks < 0) { tracing::error!( suite_id = suite.id, @@ -89,10 +88,10 @@ pub(crate) async fn decrement_incomplete_tasks( ); } - // Conditional on the state and the count, which is what makes a second - // statement safe without a transaction around the pair: a submission racing - // it either lands first and leaves a non-zero count, or lands after and sets - // `Open` itself. + // This statement has conditions on the state and on the counter. Because of this, it is safe + // as a second statement without a transaction around the two statements. A submit that occurs + // at the same time either comes first and makes the counter more than zero, or comes after and + // sets the state to `Open` itself. TaskSuites::Entity::update_many() .col_expr( TaskSuites::Column::State, @@ -109,12 +108,12 @@ pub(crate) async fn decrement_incomplete_tasks( Ok(()) } -/// The largest worker count a plan may ask an agent for. +/// The maximum number of workers that a plan can request from an agent. const MAX_WORKER_COUNT: u32 = 256; -/// Reject a plan no agent should be handed. Shared with the change path so an -/// edit cannot reach a state creation refuses. -// TODO: this should finally be adjusted to some more flexible definitions +/// Make sure that the parameters of a plan are in the permitted range. The create path and the +/// change path both use this function. +// TODO: change this check when the plan has a more flexible definition. fn validate_worker_schedule(plan: &WorkerSchedulePlan) -> Result<()> { match plan { WorkerSchedulePlan::FixedWorkers { worker_count, .. } => { @@ -128,7 +127,7 @@ fn validate_worker_schedule(plan: &WorkerSchedulePlan) -> Result<()> { Ok(()) } -/// Hooks run through the same executor as tasks, so their specs face the same check. +/// Hooks use the same executor as tasks, so their specs get the same check as task specs. fn validate_exec_hooks(hooks: &ExecHooks) -> Result<()> { for spec in [&hooks.provision, &hooks.cleanup, &hooks.background] .into_iter() @@ -223,9 +222,8 @@ pub async fn user_create_task_suite( /// Validate a suite query and resolve/authorize its target group. /// -/// Rejects empty-but-present filter sets, defaults `group_name` to the caller's -/// username when omitted, and requires the caller to have at least `Read` in the -/// resolved group. +/// The function rejects a filter set that is present and empty. If `group_name` is absent, it +/// uses the username of the caller. The caller must have at least `Read` in the group. pub(crate) async fn check_task_suites_query( user_id: i64, pool: &InfraPool, @@ -314,13 +312,12 @@ pub(crate) async fn check_task_suites_query( Ok(()) } -/// Change a suite's properties. Requires Write in the suite's group. +/// Change the properties of a suite. The caller must have `Write` in the group of the suite. /// -/// A `Cancelled` suite is frozen — effectively archived, so even a rename is -/// rejected. Every other state is editable. +/// You cannot change a `Cancelled` suite. You can change a suite in all other states. /// -/// The change reaches the *next* job. Slot counts and hooks are snapshotted when -/// a job starts, and an agent that stops tag-matching continue to run the job. +/// The change applies to the jobs that start after it, because a job copies the worker schedule +/// and the hooks when it starts. pub async fn user_change_task_suite( user_id: i64, pool: &InfraPool, @@ -335,8 +332,7 @@ pub async fn user_change_task_suite( if let Some(plan) = &req.worker_schedule { validate_worker_schedule(plan)?; } - // Only the incoming specs need checking: the hooks left alone were checked - // when they were written. + // Check only the new specs. The server checked the other hooks when it wrote them. if let Some(update) = &req.exec_hooks { for op in [&update.provision, &update.cleanup, &update.background] .into_iter() @@ -347,6 +343,9 @@ pub async fn user_change_task_suite( } } } + // If the change has new tags or a new priority, tell the agents that the suite is available. + // New tags can let an idle agent run this suite. A new priority can change which suite an idle + // agent selects or preempt an active agent. let _needs_notify = req.affects_scheduling(); let now = TimeDateTimeWithTimeZone::now_utc(); @@ -364,9 +363,9 @@ pub async fn user_change_task_suite( }; let suite_id = suite.id; - // `authorize_suite` takes no lock, so this only names the state - // in the error; the `WHERE` below is what rejects a suite - // cancelled in the meantime. + // `authorize_suite` does not lock the row, so this check only puts the state in + // the error message. The `WHERE` clause below rejects a suite that a different + // request cancels after this check. if suite.state == TaskSuiteState::Cancelled { return Err(Error::ApiError(ApiError::InvalidRequest(format!( "Cannot change a {} suite", @@ -396,16 +395,14 @@ pub async fn user_change_task_suite( if let Some(plan) = req.worker_schedule { change.worker_schedule = Set(serde_json::to_value(plan)?); } - // The hooks are a patch onto what the suite already carries, - // and merging them in Rust would mean reading the row first — - // two statements, no lock between them, and two concurrent - // patches silently losing one edit. The column is written as an - // expression over its own current value instead, so the merge - // happens inside the `UPDATE` and there is nothing to read. + // The request can set, clear or keep each hook, so the new value depends on the + // current value. If the code reads the row and merges in Rust, two changes at the + // same time can overwrite the hooks of each other. Thus, the column gets an SQL + // expression of its current value, and the merge occurs in the `UPDATE`. let hooks_expr = match req.exec_hooks { Some(update) => { - // A NULL column merges as `{}`, so a first hook lands on - // a suite that has none. + // A suite with no hooks stores NULL. Use `{}` for NULL, so that the + // first hook can be added. let mut hooks = Expr::cust_with_expr( "coalesce($1, '{}'::jsonb)", Expr::col(TaskSuites::Column::ExecHooks), @@ -416,10 +413,11 @@ pub async fn user_change_task_suite( ("background", update.background), ] { hooks = match op { - // `||` replaces this one key and leaves the rest. - // Each step parenthesises itself: `-` binds - // tighter than `||`, so an unwrapped chain would - // read as `x || (obj - 'cleanup')`. + // `||` sets this key and keeps the other keys. `-` removes it. + // + // Each step has parentheses, because `-` has a higher precedence + // than `||`. Without parentheses, PostgreSQL reads the chain as + // `x || (obj - 'cleanup')`. Some(UpdateOp::SetTo(spec)) => Expr::cust_with_exprs( "($1 || jsonb_build_object($2::text, $3::jsonb))", [ @@ -435,15 +433,13 @@ pub async fn user_change_task_suite( None => hooks, }; } - // Every hook gone is no hooks at all — the shape `create` - // stores, rather than an object of three nulls. + // If no hook stays, store NULL, not an empty object. Some(Expr::cust_with_expr("nullif($1, '{}'::jsonb)", hooks)) } None => None, }; - // `exec_hooks` is not part of `change`, so it is assigned here - // rather than twice: two assignments to one column is an error. + // `exec_hooks` is not in `change`, so this statement sets it only one time. let mut stmt = TaskSuites::Entity::update_many() .set(change) .filter(TaskSuites::Column::Id.eq(suite_id)) @@ -462,9 +458,8 @@ pub async fn user_change_task_suite( }) .await?; - // Eligibility and the pick order are read from the row on every query, so a - // change leaves nothing to rebuild — the agents it lets in just have to hear - // that the suite is there. + // Each query reads the eligibility and the pick order from the row, so there is nothing to + // rebuild. A notified agent does a new fetch and gets the latest values. // TODO: once agents can be notified, `if _needs_notify` call // `notify_suite_available(pool, _suite_id)` here. Ok(()) @@ -657,11 +652,11 @@ struct AgentUuidResult { uuid: Uuid, } -/// Get a single suite's details, including the uuids of agents allowed to execute this suite. -/// The caller must have at least `Read` in the suite's group. +/// Get the details of one suite, with the UUIDs of the agents that can run it. The caller must +/// have at least `Read` in the group of the suite. /// -/// Currently we only report agents that are UserIncluded to the suite. In the future we -/// shall read the in-memory list for actual tag matching results. +/// At this time, the result shows only the agents that a user included in the suite. +/// TODO: read the in-memory list to show the agents that match the tags. pub async fn user_get_task_suite_by_uuid( user_id: i64, pool: &InfraPool, @@ -805,9 +800,8 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U Err(ResolveError::Fatal(e)) => return Err(e), }; - // `authorize_suite` takes no lock, so this only names the state - // in the error; the `WHERE` below is what actually rejects a - // second close. + // `authorize_suite` does not lock the row, so this check only puts the state in + // the error message. The `WHERE` clause below rejects a second close. if suite.state != TaskSuiteState::Open { return Err(Error::ApiError(ApiError::InvalidRequest(format!( "Cannot transition from {} to Closed", @@ -838,13 +832,16 @@ pub async fn user_close_task_suite(user_id: i64, pool: &InfraPool, suite_uuid: U Ok(()) } -/// Cancel a suite (`* → Cancelled`, terminal). Archives every non-terminal task -/// of the suite as `Cancelled`. Requires Write/Admin in the suite's group. +/// Cancel a suite (`* → Cancelled`). `Cancelled` is a terminal state. The function archives all +/// tasks of the suite that are not in a terminal state, with the state `Cancelled`. The caller +/// must have `Write` or `Admin` in the group of the suite. /// -/// `op` (`Graceful`/`Force`) currently has no differentiated effect: its only -/// distinction is how running agents/jobs tear down and get notified, which lives -/// in the agent layer that is not ported yet. The parameter is accepted now so the -/// API stays stable; wire its behavior in with the agent work (see the seams below). +/// `op` sets what occurs to the jobs that run the suite: +/// - `Graceful`: the server tells each agent that the suite is cancelled, and which of its tasks +/// are cancelled. The agent stops taking tasks, runs the cleanup hook, and ends the job as +/// `Completed`. +/// - `Force`: the server marks the jobs `Killed` immediately, and the cleanup hook does not run. +/// The server tells the agents to stop. pub async fn user_cancel_task_suite( user_id: i64, pool: &InfraPool, @@ -874,17 +871,16 @@ pub async fn user_cancel_task_suite( )))); } - // The suite is cancelled *before* its tasks are archived, not - // after. This statement locks the suite row, and every submit - // bumps that same row under a `state <> Cancelled` predicate — so - // once this commits, a concurrent submit either got in ahead of - // the delete below or is rejected outright. Write the suite last - // instead and a submit landing between the delete and the write - // leaves a live task in a cancelled suite that nothing will ever - // claim, with `incomplete_tasks` already zeroed. + // Cancel the suite *before* the archive of its tasks, not after. This statement + // locks the suite row. Each submit updates the same row with the condition + // `state <> Cancelled`. Thus, after this commit, a submit at the same time either + // came before the delete below, or the server rejects it. If the code writes the + // suite last, a submit between the delete and the write can add a task to a + // cancelled suite. No agent will claim that task, and `incomplete_tasks` is + // already zero. // - // Zero is exact for the same reason: the delete below takes every - // active row of the suite, and nothing can add another. + // For the same reason, zero is the correct counter value. The delete below + // removes all active rows of the suite, and no request can add a new row. let fenced = TaskSuites::Entity::update_many() .col_expr( TaskSuites::Column::State, @@ -910,15 +906,14 @@ pub async fn user_cancel_task_suite( }) .inspect_err(|e| tracing::error!("{}", e))?; - // Avoid returning the rows so that we won't trigger psql's too many - // argument error when archiving many tasks. + // Do not return the rows to Rust. With many tasks, the rows cause the PostgreSQL + // error for too many arguments in the archive statement. // - // `Cancelled` rows are taken deliberately: that state means an - // agent reported `Cancel` and has not yet sent the `Commit` that - // would archive the task. Leaving them behind stalls them in - // `active_tasks` forever — the `Commit` is about to be refused by - // a terminal job, no other agent can claim a non-`Ready` task, and - // nothing reclaims that state. + // The delete also removes `Cancelled` rows, and this is intentional. This state + // means that an agent reported `Cancel`, but did not send the `Commit` that + // archives the task. If these rows stay, they stay in `active_tasks` permanently. + // The terminal job will reject the `Commit`, no other agent can claim a task that + // is not `Ready`, and no process recovers this state. let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::TaskSuiteId).eq(suite.id)) @@ -932,9 +927,9 @@ pub async fn user_cancel_task_suite( .table_name(Alias::new("deleted")) .to_owned(); - // The archived copy: every column read out of `deleted` by name, - // with `updated_at`, `state` and `result` overwritten by three - // values bound once for the whole statement. + // The archived copy. It reads each column from `deleted` by name, and replaces + // `updated_at`, `state` and `result` with three values. The statement binds each of + // these values one time. let select_from_deleted = Query::select() .expr(Expr::col(Alias::new("id"))) .expr(Expr::col(Alias::new("creator_id"))) @@ -977,28 +972,25 @@ pub async fn user_cancel_task_suite( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, - // Without this the archived copy of a suite task loses - // its suite. ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); - // Infallible here: the column list and the select have the same - // arity, which is a property of this code, not of the request. + // This cannot fail: the column list and the select have the same number of + // columns. This is a property of the code, not of the request. insert_stmt.select_from(select_from_deleted).unwrap(); let archived_cte = CommonTableExpression::new() .query(insert_stmt) .table_name(Alias::new("archived")) .to_owned(); - // The in-flight subset is the only one an agent is holding right - // now. Those agents are told which task uuids vanished, so they - // stop rather than reporting against rows we just archived. + // Agents hold only the tasks in this subset. The server tells these agents the + // UUIDs of the removed tasks. The agents then stop, and do not report on rows + // that this statement archived. // - // This reads `deleted`, not `archived`, because it needs each - // task's state as it stood *before* the archive rewrote it to - // `Cancelled`. `archived` is never referenced, and does not have - // to be: Postgres runs a data-modifying CTE exactly once and to - // completion whether or not the primary query reads its output. + // This reads `deleted`, not `archived`, because it needs the state of each task + // *before* the archive changed it to `Cancelled`. The query does not use + // `archived`, and it does not have to: PostgreSQL runs a data-modifying CTE one + // time and completely, also when the main query does not read its output. let in_flight_stmt = Query::select() .expr(Expr::col(Alias::new("uuid"))) .from(Alias::new("deleted")) @@ -1028,9 +1020,9 @@ pub async fn user_cancel_task_suite( }) .await?; - // TODO: once agents run suites, stop the jobs on `_suite_id` (`Force` kills - // them, `Graceful` lets them drain) and send each agent `SuiteCancelled`, plus - // `TasksCancelled` for `_cancelled_task_uuids`. + // TODO: when agents run suites, stop the jobs of `_suite_id`. `Force` kills the jobs, and + // `Graceful` lets them finish their current work. Send `SuiteCancelled` to each agent, and + // send `TasksCancelled` for `_cancelled_task_uuids`. Ok(()) } diff --git a/netmito/src/service/task.rs b/netmito/src/service/task.rs index fb4d740..243157a 100644 --- a/netmito/src/service/task.rs +++ b/netmito/src/service/task.rs @@ -48,8 +48,8 @@ pub(crate) fn check_exec_spec(spec: &ExecSpec) -> crate::error::Result<()> { #[derive(Clone, Copy)] enum Submitter { User, - /// A running task spawning a downstream child, identified by the parent's - /// uuid and the group the parent itself lives in. + /// A running task that submits a downstream task. It has the UUID and the group of the parent + /// task. Task { upstream_task_uuid: Uuid, parent_group_id: i64, @@ -97,18 +97,19 @@ async fn internal_submit_task( .transaction::<_, (ActiveTasks::Model, Option), crate::error::Error>( |txn| { Box::pin(async move { - // Resolve the designated suite, if any. How it is authorized - // depends on the submitter: a user reaches a suite through their - // own group memberships, while a task spawning a child may target - // any suite owned by the group the parent task itself lives in. + // Find the suite of the request, if there is one. The authorization depends on + // the submitter. A user can use a suite of a group that the user is a member + // of. A running task that submits a downstream task can use any suite of the + // group of the parent task. // - // The parent's creator is deliberately not re-checked, so a long-running suite - // keeps spawning work after that user leaves the group. + // The server does not check the creator of the parent task again, and this is + // intentional. Thus, a suite that runs for a long time can continue to submit + // tasks after that user leaves the group. let suite = match (suite_uuid, submitter) { (None, _) => None, (Some(suite_uuid), Submitter::User) => { - // Resolve the suite and the caller's membership in its - // owning group in one join. + // Find the suite and the membership of the caller in the group of the + // suite with one join. let builder = txn.get_database_backend(); let suite_stmt = Query::select() .columns([ @@ -175,8 +176,8 @@ async fn internal_submit_task( .one(txn) .await? .ok_or_else(|| { - // A suite owned by another group is reported the - // same way as one that does not exist. + // For a suite of a different group, the error is the same as + // for a suite that does not exist. Error::ApiError(crate::error::ApiError::NotFound(format!( "Suite with uuid {} in the parent task's group", suite_uuid @@ -185,10 +186,10 @@ async fn internal_submit_task( ), }; - // A first, unlocked look, so a suite that is plainly closed to - // new work is rejected with its actual state named. The - // binding check is the `WHERE` on the counter bump below — - // this one is only the better error message. + // First, do a check without a lock. It rejects a suite that does not accept new + // tasks, and puts the actual state in the error message. The `WHERE` clause on + // the counter update below is the real check. This check only gives a better + // error message. if let Some(suite) = &suite { if !suite.state.can_accept_tasks() { return Err(Error::ApiError(crate::error::ApiError::InvalidRequest( @@ -200,11 +201,10 @@ async fn internal_submit_task( } } - // The owning group, when it is already pinned by something other - // than `group_name`: a designated suite owns the task, and a - // task-spawned child always stays in its parent's group. `None` - // is the plain user path, where the group is resolved from - // `group_name` and the caller's membership. + // The group of the task, if a value other than `group_name` sets it. A suite in + // the request sets the group of the task, and a downstream task always stays in + // the group of its parent. `None` is the usual user path: the server finds the + // group from `group_name` and the membership of the caller. let pinned_group_id = match (&suite, submitter) { (Some(suite), _) => Some(suite.group_id), ( @@ -216,7 +216,7 @@ async fn internal_submit_task( (None, Submitter::User) => None, }; - // Bump the owning group's task counter atomically. + // Increase the task counter of the group in one atomic statement. let (group_id, task_id) = match pinned_group_id { Some(pinned_group_id) => { let group = Group::Entity::update_many() @@ -231,12 +231,12 @@ async fn internal_submit_task( .into_iter() .next() .ok_or_else(|| { - // The row must exist: both the suite and the - // parent task hold an FK to it. + // The row must exist, because the suite and the parent task + // both have a foreign key to it. tracing::error!("Owning group {} not found", pinned_group_id); Error::ApiError(crate::error::ApiError::InternalServerError) })?; - // The request's group_name must match + // The `group_name` of the request must be the same group. if group.group_name != group_name { let owner = match &suite { Some(suite) => { @@ -260,19 +260,20 @@ async fn internal_submit_task( (group.id, group.task_count) } None => { - // Resolve the user-specified group, verify the caller's - // membership, and bump its task counter in one statement. - // No row back means either the group doesn't exist or the - // caller isn't in it; both return 404 so users won't know - // about the existence of a group they are not a member of. - // Membership must stay a subquery, not a join: both table has a - // `id` column and sea-orm will create a unqualified `RETURNING id`, - // which Postgres rejects as ambiguous. + // Find the group of the request, make sure that the caller is a member, + // and increase the task counter of the group, all in one statement. If + // no row comes back, the group does not exist or the caller is not a + // member. The two cases return 404, so that a user cannot find out if a + // group exists when the user is not a member. + // The membership check must be a subquery, not a join. The two tables + // have an `id` column, and sea-orm makes an unqualified `RETURNING id`. + // PostgreSQL rejects it as ambiguous. let member_of = Query::select() .column(UserGroup::Column::GroupId) .from(UserGroup::Entity) .and_where(Expr::col(UserGroup::Column::UserId).eq(creator_id)) - // TODO: when there is 'exec' access level, enforce access check here. + // TODO: when there is an 'exec' access level, do the access check + // here. // // .and_where( // Expr::col(UserGroup::Column::Role) @@ -301,17 +302,16 @@ async fn internal_submit_task( } }; - // One statement, so both increments are evaluated server-side - // on the row this `UPDATE` itself locked. Read the counters - // first and write literals back and two concurrent submits - // both see `total = n` and both write `n + 1`: the suite - // undercounts, `decrement_incomplete_tasks` reaches zero with - // tasks still `Ready`, and the surplus is stranded in a - // `Complete` suite that hands out nothing. + // Use one statement, so that the database calculates the two increases on the + // row that this `UPDATE` locks. If the code reads the counters first and writes + // the values back, two submits at the same time both read `total = n` and both + // write `n + 1`. Then the counter of the suite is too low, and + // `decrement_incomplete_tasks` gets to zero while tasks are still `Ready`. + // These tasks then stay in a `Complete` suite that gives out no tasks. // - // The state predicate carries the accepts-tasks check, so no - // row is bumped after a cancellation that committed since the - // read above. + // The condition on the state does the check that the suite accepts tasks. Thus, + // the statement does not update the row if a cancel committed after the read + // above. let suite = match suite { None => None, Some(suite) => Some( @@ -344,7 +344,7 @@ async fn internal_submit_task( .into_iter() .next() .ok_or_else(|| { - // Cancelled between the read above and here. + // A cancel committed after the read above. Error::ApiError(crate::error::ApiError::InvalidRequest( format!("Suite {} stopped accepting new tasks", suite.uuid), )) @@ -375,18 +375,17 @@ async fn internal_submit_task( }; let task = task.insert(txn).await?; - // Queued here, inside the transaction, and never after it - // commits. The queue is emptied by `served`, which removes - // what a claim locked — so a task pushed before its commit - // is always pushed before any `served` that could take it - // back out, because no claim can lock a row it cannot yet - // see. Push after the commit instead and a claim landing in - // between wins the race: it takes the task, `served` finds - // nothing in the queue to remove, and the push then puts an - // already-running task back on the suite's backlog, where - // nothing will ever remove it again. - // TODO: once suites have a ready queue, push a `Ready` suite task - // onto it here with `queues.push_ready(suite.id, [(task.id, task.priority)])`. + // Push the task to the queue here, in the transaction, and not after the + // commit. `served` empties the queue: it removes the tasks that a claim locked. + // A claim cannot lock a row before the row is committed. Thus, a task that is + // pushed before its commit is always pushed before a `served` call that can + // remove it. If the push occurs after the commit, a claim can come between the + // commit and the push. The claim takes the task, and `served` finds nothing to + // remove. Then the push adds a running task to the backlog of the suite, and no + // process removes it. + // + // TODO: when suites have a ready queue, push a `Ready` suite task to it here + // with `queues.push_ready(suite.id, [(task.id, task.priority)])`. Ok((task, suite)) }) @@ -457,12 +456,11 @@ pub async fn user_submit_task( internal_submit_task(pool, creator_id, Submitter::User, req).await } -/// Submit the downstream child of a running task, on behalf of the worker or -/// agent executing it. +/// Submit a downstream task of a running task, for the worker or agent that runs the parent task. /// -/// The child stays in the parent's group, and `req.suite_uuid` may name any -/// suite that group owns — the parent's own suite, a sibling one, or `None` for -/// a suite-less task dispatched to workers. +/// The downstream task stays in the group of its parent. `req.suite_uuid` can be any suite of that +/// group: the suite of the parent, a different suite, or `None` for a task without a suite that +/// goes to workers. pub async fn worker_submit_pending_task( pool: &InfraPool, creator_id: i64, @@ -496,11 +494,11 @@ pub async fn worker_trigger_pending_task(pool: &InfraPool, uuid: Uuid) -> crate: task.state = Set(TaskState::Ready); task.updated_at = Set(TimeDateTimeWithTimeZone::now_utc()); let task = task.update(&pool.db).await?; - // A suite task never enters the worker queues — agents pull it from its - // suite. Wake the idle eligible agents instead. + // A suite task does not go into the worker queues. Agents get it from its suite. Thus, wake the + // idle agents that can run the suite. if let Some(_suite_id) = task.task_suite_id { - // TODO: once agents can be notified, call - // `notify_suite_available(pool, _suite_id)` here. + // TODO: when agents can get notifications, call `notify_suite_available(pool, _suite_id)` + // here. return Ok(()); } // Batch add task to worker task queues @@ -780,10 +778,9 @@ pub async fn user_cancel_task( downstream_task_uuid: Set(task.downstream_task_uuid), task_suite_id: Set(task.task_suite_id), }; - // A cancelled suite task never reaches an agent `Commit`, so - // this is the only chance to give the suite its count back. It - // goes first so the suite row is taken before the task rows — - // the order every writer of the pair uses. + // A cancelled suite task does not get to an agent `Commit`. Thus, this is the only + // point where the suite can get its count back. This step comes first, so that the + // suite row is locked before the task rows. All writers use this order. if let Some(suite_id) = suite_id { crate::service::suite::decrement_incomplete_tasks(txn, suite_id, 1, now) .await?; @@ -796,8 +793,7 @@ pub async fn user_cancel_task( .await?; let _ = remove_task(task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); - // TODO: once suites have a ready queue, remove a cancelled suite task from it - // as well. + // TODO: when suites have a ready queue, also remove a cancelled suite task from it. Ok(()) } @@ -846,8 +842,7 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((ActiveTasks::Entity, ActiveTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) - // Left join: a task without a suite is the common case, and those - // rows have to survive it. + // Use a left join: most tasks do not have a suite, and the query must keep these rows. .join( sea_orm::JoinType::LeftJoin, TaskSuites::Entity, @@ -909,8 +904,7 @@ pub async fn get_task_by_uuid(pool: &InfraPool, uuid: Uuid) -> crate::error::Res Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) - // Left join: a task without a suite is the common case, and those - // rows have to survive it. + // Use a left join: most tasks do not have a suite, and the query must keep these rows. .join( sea_orm::JoinType::LeftJoin, TaskSuites::Entity, @@ -971,15 +965,15 @@ struct UserGroupRoleQueryRes { role: UserGroupRole, } -/// The suite a query is restricted to, resolved together with the group that -/// owns it so the two can be checked against each other. +/// The suite that limits a query, together with the group of the suite, so that the code can +/// compare the two. #[derive(FromQueryResult)] struct SuiteIdWithGroup { id: i64, group_name: String, } -/// Returns the id of the suite the query names, for [`apply_task_filters`]. +/// Returns the id of the suite in the query, for [`apply_task_filters`]. pub(crate) async fn check_task_list_query( user_id: i64, pool: &InfraPool, @@ -1014,13 +1008,13 @@ pub(crate) async fn check_task_list_query( ))); } } - // A suite belongs to exactly one group, and submitting to it pins the task - // to that same group, so the suite names the group rather than narrowing - // it: it fills in a group the caller left out, and contradicts one they got - // wrong. Resolved through the caller's own memberships, so a suite in a - // group they are not in, one that does not exist, and one that disagrees - // with `group_name` are all the same answer — and that answer never names - // the owning group, which would otherwise leak out of a bare uuid. + // A suite is in exactly one group, and a task submitted to the suite gets that group. Thus, the + // suite identifies the group, and does not make the group filter smaller. If the caller gives + // no group, the server uses the group of the suite. If the caller gives a different group, the + // query fails. The server finds the suite through the memberships of the caller. Thus, the + // answer is the same for a suite in a group that the caller is not in, a suite that does not + // exist, and a suite that is not in `group_name`. The answer does not show the group of the + // suite, because a UUID alone must not show it. let suite_id = match query.suite_uuid { None => None, Some(suite_uuid) => { @@ -1051,16 +1045,16 @@ pub(crate) async fn check_task_list_query( .one(&pool.db) .await? .ok_or_else(|| { - // This fires when there is no query result, i.e. the suite does not exist or - // the user does not have access to it + // This occurs when the query has no result: the suite does not exist, or the + // user cannot access it. Error::ApiError(crate::error::ApiError::NotFound(format!( "User doesn't have permission or suite with uuid {suite_uuid}" ))) })?; match query.group_name { Some(ref group_name) if *group_name != suite.group_name => { - // This fires then the user's specified group_name does not match - // the actual owner group of the suite. + // This occurs when the `group_name` of the request is not the group of the + // suite. return Err(Error::ApiError(crate::error::ApiError::NotFound(format!( "User doesn't have permission or suite with uuid {suite_uuid}" )))); @@ -1160,9 +1154,9 @@ pub(crate) fn apply_task_filters( query: &TasksQueryReq, suite_id: Option, ) -> crate::error::Result<()> { - // The id rather than `query.suite_uuid`: `check_task_list_query` had to read - // the row anyway to authorize it, and the archive keeps the same column, so - // neither statement needs the suite table joined. + // Use the id, not `query.suite_uuid`. `check_task_list_query` read the row for the + // authorization, and the archive has the same column. Thus, the two statements do not need a + // join on the suite table. if let Some(suite_id) = suite_id { active_stmt.and_where( Expr::col((ActiveTasks::Entity, ActiveTasks::Column::TaskSuiteId)).eq(suite_id), @@ -1421,8 +1415,7 @@ pub async fn query_tasks_by_filter( Expr::col((ActiveTasks::Entity, ActiveTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) - // Left join: a task without a suite is the common case, and those - // rows have to survive it. + // Use a left join: most tasks do not have a suite, and the query must keep these rows. .join( sea_orm::JoinType::LeftJoin, TaskSuites::Entity, @@ -1487,8 +1480,7 @@ pub async fn query_tasks_by_filter( Expr::col((ArchivedTasks::Entity, ArchivedTasks::Column::GroupId)) .eq(Expr::col((Group::Entity, Group::Column::Id))), ) - // Left join: a task without a suite is the common case, and those - // rows have to survive it. + // Use a left join: most tasks do not have a suite, and the query must keep these rows. .join( sea_orm::JoinType::LeftJoin, TaskSuites::Entity, @@ -1540,11 +1532,11 @@ struct IdResult { task_suite_id: Option, } -/// Tick `incomplete_tasks` down once per cancelled task, for every suite a -/// batch cancel touched. +/// Decrease `incomplete_tasks` by one for each cancelled task, for each suite that a batch cancel +/// changed. /// -/// The rows come straight from the archive's `RETURNING`, so this counts what -/// was actually cancelled rather than what was asked for. +/// The rows come directly from the `RETURNING` of the archive. Thus, the count is the number of +/// tasks that the cancel really removed, not the number in the request. async fn decrement_incomplete_tasks_by_suite( txn: &C, suite_ids: impl IntoIterator>, @@ -1554,11 +1546,10 @@ async fn decrement_incomplete_tasks_by_suite( for suite_id in suite_ids.into_iter().flatten() { *per_suite.entry(suite_id).or_default() += 1; } - // Sorted, not iterated straight off the map: each decrement writes its - // suite row and holds that lock to commit, so two cancels over overlapping - // suites in opposite orders deadlock. The `locked` CTE of each batch cancel - // has normally taken these rows already, which makes this a backstop — it - // still matters for the straggler suite that CTE cannot cover. + // Sort the suites, and do not use the map order. Each decrease writes its suite row and keeps + // the lock until the commit. If two cancels change the same suites in opposite order, they + // deadlock. Usually the `locked` CTE of each batch cancel already locked these rows, so this + // sort is a second safety step. It is still necessary for a suite that the CTE cannot lock. let mut per_suite: Vec<(i64, i32)> = per_suite.into_iter().collect(); per_suite.sort_unstable_by_key(|(suite_id, _)| *suite_id); for (suite_id, count) in per_suite { @@ -1644,18 +1635,17 @@ pub async fn cancel_tasks_by_filter( // Build the complete CTE statement before the transaction to avoid lifetime issues - // The suite rows this cancel will decrement. Every other writer of the pair - // takes the suite row before the task rows, and this is how a batch cancel - // joins them — on its own it learns its suites from the archive's `RETURNING`, - // far too late to lock anything. + // The suite rows that this cancel will decrease. All other writers lock the suite row before + // the task rows, and a batch cancel uses this CTE to do the same. Without it, the cancel finds + // its suites only from the `RETURNING` of the archive, and that is too late to lock them. // - // In the same statement as the delete, so one snapshot covers both: the filter - // selects the same tasks in each CTE, and a task submitted in between is - // invisible to either. + // This CTE is in the same statement as the delete, so one snapshot applies to the two. The + // filter selects the same tasks in each CTE, and the CTEs do not see a task that is submitted + // between them. // - // `ORDER BY id` is load-bearing. Postgres puts `LockRows` above the sort, so - // rows are locked in id order; any scan is therefore a prefix of ascending - // ids, and overlapping cancels cannot take two rows in opposite orders. + // `ORDER BY id` is necessary. PostgreSQL puts `LockRows` above the sort, so it locks the rows + // in id order. Thus, each scan locks rows in ascending id order, and two cancels cannot lock + // two rows in opposite order. let locked_suites = Query::select() .column(TaskSuites::Column::Id) .from(TaskSuites::Entity) @@ -1682,10 +1672,9 @@ pub async fn cancel_tasks_by_filter( let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::Id).in_subquery(task_id_subquery)) - // This predicate is the guarantee, not a nicety: a task is deleted only if - // its suite came out of `locked`, and producing that row is what locked it. - // So the suite row is always taken before the task row, whatever plan the - // subquery gets. + // This condition is the guarantee, not an extra check. The statement deletes a task only if + // its suite is in `locked`, and `locked` locks each row that it produces. Thus, the suite + // row is always locked before the task row, for all query plans of the subquery. .and_where( Expr::col(ActiveTasks::Column::TaskSuiteId) .is_null() @@ -1770,8 +1759,8 @@ pub async fn cancel_tasks_by_filter( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, - // Without this the archived copy of a suite task loses - // its suite, and the count below has nothing to key on. + // Without this column, the archived copy of a suite task does not have its + // suite, and the count below cannot group the tasks by suite. ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); @@ -1803,8 +1792,7 @@ pub async fn cancel_tasks_by_filter( let _ = remove_task(*task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); } - // TODO: once suites have a ready queue, remove the cancelled suite tasks - // from it as well. + // TODO: when suites have a ready queue, also remove the cancelled suite tasks from it. Ok(TasksCancelByFilterResp { cancelled_count: task_ids.len() as u64, @@ -1880,18 +1868,18 @@ pub async fn cancel_tasks_by_uuids( // Build CTE for DELETE RETURNING + INSERT SELECT to avoid parameter limits // Convert the SELECT statement into a subquery for the DELETE - // The suite rows this cancel will decrement. Every other writer of the pair - // takes the suite row before the task rows, and this is how a batch cancel - // joins them — on its own it learns its suites from the archive's `RETURNING`, - // far too late to lock anything. + // The suite rows that this cancel will decrease. All other writers lock the suite row + // before the task rows, and a batch cancel uses this CTE to do the same. Without it, the + // cancel finds its suites only from the `RETURNING` of the archive, and that is too late to + // lock them. // - // In the same statement as the delete, so one snapshot covers both: the filter - // selects the same tasks in each CTE, and a task submitted in between is - // invisible to either. + // This CTE is in the same statement as the delete, so one snapshot applies to the two. The + // filter selects the same tasks in each CTE, and the CTEs do not see a task that is + // submitted between them. // - // `ORDER BY id` is load-bearing. Postgres puts `LockRows` above the sort, so - // rows are locked in id order; any scan is therefore a prefix of ascending - // ids, and overlapping cancels cannot take two rows in opposite orders. + // `ORDER BY id` is necessary. PostgreSQL puts `LockRows` above the sort, so it locks the + // rows in id order. Thus, each scan locks rows in ascending id order, and two cancels + // cannot lock two rows in opposite order. let locked_suites = Query::select() .column(TaskSuites::Column::Id) .from(TaskSuites::Entity) @@ -1918,10 +1906,10 @@ pub async fn cancel_tasks_by_uuids( let delete_stmt = DeleteStatement::new() .from_table(ActiveTasks::Entity) .and_where(Expr::col(ActiveTasks::Column::Id).in_subquery(id_subquery)) - // This predicate is the guarantee, not a nicety: a task is deleted only if - // its suite came out of `locked`, and producing that row is what locked it. - // So the suite row is always taken before the task row, whatever plan the - // subquery gets. + // This condition is the guarantee, not an extra check. The statement deletes a task + // only if its suite is in `locked`, and `locked` locks each row that it produces. Thus, + // the suite row is always locked before the task row, for all query plans of the + // subquery. .and_where( Expr::col(ActiveTasks::Column::TaskSuiteId) .is_null() @@ -2004,9 +1992,8 @@ pub async fn cancel_tasks_by_uuids( ArchivedTasks::Column::Result, ArchivedTasks::Column::UpstreamTaskUuid, ArchivedTasks::Column::DownstreamTaskUuid, - // Without this the archived copy of a suite task - // loses its suite, and the count below has nothing - // to key on. + // Without this column, the archived copy of a suite task does not have + // its suite, and the count below cannot group the tasks by suite. ArchivedTasks::Column::TaskSuiteId, ]) .to_owned(); @@ -2045,8 +2032,7 @@ pub async fn cancel_tasks_by_uuids( let _ = remove_task(*task_id, pool) .inspect_err(|e| tracing::warn!("Failed to remove task {}: {:?}", task_id, e)); } - // TODO: once suites have a ready queue, remove the cancelled suite tasks - // from it as well. + // TODO: when suites have a ready queue, also remove the cancelled suite tasks from it. // Determine which UUIDs failed (not found or no permission) let failed_uuids: Vec = req diff --git a/netmito/src/service/worker/mod.rs b/netmito/src/service/worker/mod.rs index 6ea1f59..6660b0b 100644 --- a/netmito/src/service/worker/mod.rs +++ b/netmito/src/service/worker/mod.rs @@ -642,9 +642,8 @@ pub async fn report_task( if user.state != UserState::Active { return Err(AuthError::PermissionDenied.into()); } - // Verify worker has permission to access this group. The child's own - // group and suite are pinned to the parent's by the submit path, - // which also rejects a mismatched `req.group_name`. + // Make sure that the worker can access this group. The submit path gives the + // downstream task the group of its parent, and rejects a different `req.group_name`. GroupWorker::Entity::find() .filter(GroupWorker::Column::WorkerId.eq(worker_id)) .filter(GroupWorker::Column::GroupId.eq(task.group_id)) @@ -714,12 +713,11 @@ pub async fn report_task( ..Default::default() }; let suite_id = task.task_suite_id; - // The suite's counter goes down in the same transaction that - // archives the task. Split across two, a failure in between leaves - // `incomplete_tasks` permanently too high, and a suite that never - // reaches zero never completes. It goes down *first* so this takes - // the suite row before the task rows, the order every writer of the - // pair uses. + // Decrease the suite counter in the same transaction that archives the task. + // With two transactions, a failure between them keeps `incomplete_tasks` too + // high permanently. Then the counter never gets to zero, and the suite never + // completes. Decrease the counter first. This locks the suite row before the + // task rows, which is the lock order that all writers use. let (worker_id, worker_state) = pool .db .transaction::<_, (i64, WorkerState), Error>(|txn| { diff --git a/openapi.yaml b/openapi.yaml index 7f7bab9..ded4a20 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1647,15 +1647,14 @@ paths: put: summary: Change task suite description: >- - Change a suite's properties. Every field is optional and an omitted one is - left unchanged; an empty request is rejected. Nullable fields (name, - description and each hook) take an update op rather than a bare value, so - clearing one is expressible. A Cancelled suite is frozen and rejects every - change, including a rename. The change applies to the jobs started after - it: an agent mid-job keeps the worker count and hooks it started with, and - one that no longer tag-matches still finishes the job it holds. The caller - must have the Write or Admin role in the suite's group; a non-member and an - unknown uuid are indistinguishable and both return 404. + Change the properties of a suite. All fields are optional, and an absent field + does not change. The server rejects an empty request. A nullable field (name, + description and each hook) takes an update op, not a plain value, so that you can + clear it. You cannot change a Cancelled suite, and this includes its name. The + change applies to the jobs that start after it, because a job copies the worker + schedule and the hooks when it starts. The caller must have the Write or Admin + role in the group of the suite. If the caller is not a member, or the UUID does + not exist, the server returns 404 for the two cases. operationId: changeTaskSuite tags: - Suites @@ -3325,10 +3324,11 @@ components: format: uuid nullable: true description: >- - Filter by the task suite. A suite names its own group, so it fills in - group_name when that is omitted and must agree with it otherwise. A - suite that does not exist, is in a group the caller is not in, or - disagrees with group_name is a 404, not an empty result. + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array @@ -3415,7 +3415,7 @@ components: type: string format: uuid nullable: true - description: The task suite that owns this task, if any + description: The task suite of this task, if the task is in a suite task_id: type: integer format: int64 @@ -3500,7 +3500,7 @@ components: type: string format: uuid nullable: true - description: The task suite that owns this task, if any + description: The task suite of this task, if the task is in a suite task_id: type: integer format: int64 @@ -3744,10 +3744,11 @@ components: format: uuid nullable: true description: >- - Filter by the task suite. A suite names its own group, so it fills in - group_name when that is omitted and must agree with it otherwise. A - suite that does not exist, is in a group the caller is not in, or - disagrees with group_name is a 404, not an empty result. + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array @@ -3944,10 +3945,11 @@ components: format: uuid nullable: true description: >- - Filter by the task suite. A suite names its own group, so it fills in - group_name when that is omitted and must agree with it otherwise. A - suite that does not exist, is in a group the caller is not in, or - disagrees with group_name is a 404, not an empty result. + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array @@ -4132,10 +4134,11 @@ components: format: uuid nullable: true description: >- - Filter by the task suite. A suite names its own group, so it fills in - group_name when that is omitted and must agree with it otherwise. A - suite that does not exist, is in a group the caller is not in, or - disagrees with group_name is a 404, not an empty result. + Show only the tasks of this suite. The suite also identifies the group. If + group_name is absent, the server uses the group of the suite. If group_name is + present, it must be the group of the suite. The server returns 404, not an + empty result, for a suite that does not exist, a suite in a group that the + caller is not in, and a suite that is not in group_name. example: 789e0123-e89b-12d3-a456-426614174222 tags: type: array @@ -4465,8 +4468,8 @@ components: UpdateOpString: type: object description: >- - A change to a nullable string field. `set_to` writes the value, `set_null` - clears it; omitting the field from the request leaves it unchanged. + A change to a nullable string field. `set_to` writes the value, and `set_null` + clears it. If the request does not contain the field, the field does not change. required: - op properties: @@ -4477,7 +4480,7 @@ components: - set_null value: type: string - description: Required when op is set_to, absent otherwise + description: Necessary when op is set_to. Absent for the other ops example: op: set_to value: nightly-eval @@ -4485,8 +4488,8 @@ components: UpdateOpExecSpec: type: object description: >- - A change to one lifecycle hook. `set_to` installs the spec, `set_null` - removes the hook; omitting the field leaves that hook as it was. + A change to one lifecycle hook. `set_to` sets the spec, and `set_null` removes the + hook. If the request does not contain the field, the hook does not change. required: - op properties: @@ -4497,13 +4500,14 @@ components: - set_null value: type: object - description: Execution spec, required when op is set_to + description: The execution spec. Necessary when op is set_to ExecHooksUpdate: type: object description: >- - Per-hook changes. Each hook is set, cleared, or — omitted — left alone; - clearing all three stores no hooks at all. + The changes to the hooks, one for each hook. You can set a hook, clear a hook, or + leave a hook out to keep it. If you clear all three hooks, the suite stores no + hooks. properties: provision: $ref: "#/components/schemas/UpdateOpExecSpec" @@ -4514,7 +4518,7 @@ components: ChangeTaskSuiteReq: type: object - description: At least one field must be present, or the request is rejected as empty + description: At least one field must be present. The server rejects an empty request properties: name: $ref: "#/components/schemas/UpdateOpString" @@ -4525,8 +4529,9 @@ components: items: type: string description: >- - Replacement tags for agent matching. An empty array matches every - agent the suite's group can write to, since matching is containment + The new tags that an agent must have to run the suite. An empty array matches + all agents that the group of the suite can write to, because the match is a + containment check example: - linux - cuda:11 @@ -4534,7 +4539,7 @@ components: type: array items: type: string - description: Replacement labels for querying/filtering + description: The new labels, which you can use to query and filter suites priority: type: integer format: int32 @@ -4543,8 +4548,8 @@ components: allOf: - $ref: "#/components/schemas/WorkerSchedulePlan" description: >- - Replaces the plan wholesale; a patch could not express switching to - another scheduling policy + Replaces the full plan. A partial change cannot change the plan to a different + scheduling policy exec_hooks: $ref: "#/components/schemas/ExecHooksUpdate"