diff --git a/app/application.ml b/app/application.ml index e573c788..6a696d64 100644 --- a/app/application.ml +++ b/app/application.ml @@ -1179,13 +1179,15 @@ let back_state state = { state with routes; timeline } ;; -let coordinated_error_message = function - | Journal_worker_coordination.Worker_api.Db_error error -> Logseq_db_worker.Error.message error - | Invalid_input message | Service_failed message | Unsupported_operation message - | Protocol_mismatch message | Stale_read message | Resync_required message -> message - | Revoked_session -> "The graph session changed." - | Host_change_in_progress -> "The graph is closing. The outcome is retained." - | Response_budget_exceeded -> "The result exceeded its response budget." +let coordinated_error_message = Journal_worker_coordination_host.error_message +;; +let report_coordination_error state ~operation ~phase ?recovery error = + let error=Journal_worker_coordination_host.diagnostic_error error in + let latest=List.find_opt(fun (entry:worker_error_occurrence)-> + entry.active && entry.operation=operation && entry.phase=Some phase + && entry.graph_generation=state.graph_state.generation)state.worker_errors in + if Option.exists(fun entry->entry.error=error)latest then state + else record_worker_error state ~operation ~phase ?recovery error ;; let owns_capture state command = Option.bind state.direct_capture Journal_capture.pending_command=Some command @@ -1198,6 +1200,8 @@ let rec apply_worker_response ?(settling=false) ~discard state (response : Journ | Block_captured{timeline_entry_update=Some entry;_}-> Journal_timeline_state.prepend_timeline_entry state.timeline entry | _->state.timeline in + let state=Option.fold ~none:state ~some:(report_coordination_error state + ~operation:"appendJournal" ~phase:"presentation") failure in let message=Option.map coordinated_error_message failure in {state with timeline;direct_capture=Option.map Journal_capture.lock_for_retry state.direct_capture; capture_error=Option.map(fun message->Local_capture_failure message)message; @@ -1217,16 +1221,19 @@ let rec apply_worker_response ?(settling=false) ~discard state (response : Journ (match failure with | Some error when not insert_rejected-> let state=apply_worker_response ~settling:true ~discard state {payload=result}in - report_ui_error state ~operation:"captureTaskStatus" ~phase:"mutation" - ~message:(coordinated_error_message error) ~recovery:"The stored entry is retained." + report_coordination_error state ~operation:"captureTaskStatus" ~phase:"mutation" + ~recovery:"The stored entry is retained." error | Some error->let message=coordinated_error_message error in + let state=report_coordination_error state ~operation:"appendJournal" ~phase:"mutation" error in {state with direct_capture=Option.map(fun capture-> Journal_capture.release_rejected_attempt capture)state.direct_capture; capture_error=Some(Local_capture_failure message);capture_error_owner=Some Timeline_composer} | None->apply_worker_response ~settling:true ~discard state {payload=result}) | Child_projection_ready {command;result;failure;_} -> if not(Journal_routes.has_child_request state.routes command) then state - else (match result with + else (let state=Option.fold ~none:state ~some:(report_coordination_error state + ~operation:"insertChild" ~phase:"presentation") failure in + match result with | Child_created{child;parent;_}-> {state with routes=Journal_routes.preview_child_created state.routes ~command ~child ~parent ~error:(Option.map coordinated_error_message failure)} @@ -1246,12 +1253,13 @@ let rec apply_worker_response ?(settling=false) ~discard state (response : Journ if not(Journal_routes.has_child_request state.routes command) then state else (match failure with | Some failure when insert_rejected -> + let state=report_coordination_error state ~operation:"insertChild" ~phase:"mutation" failure in {state with routes=Journal_routes.release_rejected_child state.routes ~command ~message:(coordinated_error_message failure)} | Some failure -> let state=apply_worker_response ~settling:true ~discard state {payload=result}in - report_ui_error state ~operation:"childTaskStatus" ~phase:"mutation" - ~message:(coordinated_error_message failure) ~recovery:"The stored child and attachments are retained." + report_coordination_error state ~operation:"childTaskStatus" ~phase:"mutation" + ~recovery:"The stored child and attachments are retained." failure | None->apply_worker_response ~settling:true ~discard state {payload=result}) | Mutation_settled(intent,presentation,payload)-> let next=apply_worker_response ~settling:true ~discard state {payload} in @@ -1724,6 +1732,7 @@ let rec apply_worker_response ?(settling=false) ~discard state (response : Journ ; timeline_notice = Some (Delete_failed message) } | _ -> state)) + | Diagnostic failure -> record_runtime_worker_failure state failure | Rejected failure -> let state = record_failure_source state failure in show_sync_error @@ -1743,7 +1752,16 @@ let application_error_rows errors = ; "Code", Logseq_db_worker.Error.code_string (Logseq_db_worker.Error.code occurrence.error) ; "Scope", Printf.sprintf "graph generation=%d" occurrence.graph_generation ; "Recovery", occurrence.recovery - ]) + ] @ + let trace=Logseq_db_worker.Error.trace occurrence.error in + let rows prefix (cause:Logseq_db_worker.Error.cause)= + [prefix ^ " component",Logseq_db_worker.Error.component_string cause.component; + prefix ^ " operation",cause.operation; + prefix ^ " code",Option.value cause.code ~default:"unknown"] + @ (if String.equal cause.message (Logseq_db_worker.Error.message occurrence.error) + then [] else [prefix ^ " reason",cause.message]) in + List.concat(List.mapi(fun i cause->rows ("Context " ^ string_of_int(i+1)) cause)trace.contexts) + @ rows "Origin" trace.origin) errors ;; @@ -3170,30 +3188,29 @@ let dispose_context context = context.running := false; current_app := None; Worker.Private.request_stop context.client; - (try context.release_staging () with _ -> - report_cleanup_warning "journal: operation=releaseDraftStaging code=cleanupPending\n"); - (try ignore (Lui_app.dispose context.app) with _ -> - report_cleanup_warning "journal: operation=disposeApplication code=cleanupPending\n") + let warning operation error = + let module D=Logseq_db_types.Failure_diagnostic in + let detail=D.of_source ~operation:"closeDatabase" ~reason:"cleanupFailed" (Printexc.to_string error) in + report_cleanup_warning ("journal: operation=" ^ operation ^ " " ^ D.message detail ^ "\n") in + (try context.release_staging () with error -> warning "releaseDraftStaging" error); + (try ignore (Lui_app.dispose context.app) with error -> warning "disposeApplication" error) ;; let decode_extension_values payload = - let json = - try Yojson.Safe.from_string payload with - | _ -> `Null - in - match json with - | `Assoc fields -> - List.fold_left - (fun map (key, value) -> - match value with - | `String s -> Lui_protocol.String_map.add key (Lui_protocol.StringValue s) map - | `Bool b -> Lui_protocol.String_map.add key (Lui_protocol.BoolValue b) map - | `Int i -> Lui_protocol.String_map.add key (Lui_protocol.IntValue i) map - | `Float f -> Lui_protocol.String_map.add key (Lui_protocol.FloatValue f) map - | _ -> map) - Lui_protocol.String_map.empty - fields - | _ -> Lui_protocol.String_map.empty + let decode = function + | `Assoc fields -> + List.fold_left(fun result (key,value)->Result.bind result(fun map-> + let value=match value with + | `String text->Ok(Lui_protocol.StringValue text) + | `Bool flag->Ok(Lui_protocol.BoolValue flag) + | `Int number->Ok(Lui_protocol.IntValue number) + | `Float number->Ok(Lui_protocol.FloatValue number) + | _->Error "invalid JSON value" in + Result.map(fun value->Lui_protocol.String_map.add key value map)value)) + (Ok Lui_protocol.String_map.empty)fields + | _->Error "Native event shape is invalid" in + try Logseq_db_types.Json_codec.decode decode (Yojson.Safe.from_string payload) + with Yojson.Json_error _->Error "Native event JSON syntax is invalid" ;; (* Platform request tags map to the tag carried by their response envelope; @@ -3421,8 +3438,8 @@ let start List.find_opt(Wc.matches_asset_scope session)manager.resident_scopes)in Option.iter(fun scope->run_owned(Some(timeline_owner_of_scope scope))(fun()-> match !set_state_ref with None->Effect.ignore|Some set_state->set_state(fun state-> - report_ui_error state ~operation:"recoverImportSource" ~phase:"readback" - ~message:(coordinated_error_message error) ~recovery:"The original attachment input is retained.")))scope))!coordination_host; + report_coordination_error state ~operation:"recoverImportSource" ~phase:"readback" + ~recovery:"The original attachment input is retained." error)))scope))!coordination_host; Rrbvec.iter(fun recipient-> Rrbvec.iter(fun intent-> Option.iter(fun proof->Coordination_host.post recipient(Coordination.Import_source_reconciled proof)) @@ -3503,7 +3520,7 @@ let start !mutation_acknowledgements_changed(); !retained_insertions_changed()); !coordination_lifecycle_changed()) with - | Error _->Error "Unable to bind the graph reader. Try Copy again." + | Error error->Error(coordinated_error_message error) | Ok host->coordination_host_id:=Some id;coordination_host:=Some(identity,host);Ok host) in let route_coordination_event event = @@ -3642,7 +3659,7 @@ let start ~maximum_bytes:(Wc.capabilities session).limits.response_budget_bytes in let requested=Result.bind budget(fun budget->Coordination_host.request host session(Wc.Query(budget,query))) in match requested with - | Error _->fail "The copy read could not be admitted. Try again." + | Error error->fail(coordinated_error_message error) | Ok ticket-> copy_cancel:=Some(fun()->Coordination_host.post host(Coordination.Cancel ticket); Coordination_host.post host(Coordination.Forget ticket)); @@ -3655,10 +3672,11 @@ let start | Wc.Answered(Ok value)->Ok(convert value) | Answered(Error(Response_budget_exceeded))->Error "The selected block is too large to copy." | Answered(Error(Stale_read _ | Resync_required _ | Revoked_session))->Error "The graph changed while copying. Try again." - | Answered(Error _)|Not_admitted _|Interrupted _->Error "The copy read failed or was interrupted. Try again." in + | (Answered(Error _)|Not_admitted _|Interrupted _) as failed-> + Error(Coordination_host.reply_error_message ~fallback:"The copy read failed." failed) in completed result)) | Awaiting_baseline|Bootstrap_not_ready _->() - | Bootstrap_failed _|Binding_revoked _->fail "The graph reader could not prepare. Try Copy again." + | Bootstrap_failed error|Binding_revoked error->fail(coordinated_error_message error) in copy_poll:=Some await; let view=Coordination_host.view host in @@ -3888,7 +3906,11 @@ let start (!media_changes_cell)(Ok()) in consumed,application,result)) ~f:(fun(consumed,application,result)-> - let flush=match !set_state_ref with None->Effect.ignore|Some set_state->flush_media set_state in + let flush=match !set_state_ref with None->Effect.ignore|Some set_state-> + Effect.bind (match consumed,result with + |true,Error error->set_state(fun state->report_coordination_error state + ~operation:"readMedia" ~phase:"read" error) + |_->Effect.ignore) ~f:(fun()->flush_media set_state) in Effect.bind flush ~f:(fun()-> Option.iter(fun token->Option.iter(fun(_,host)->Coordination_host.post host (if consumed then Coordination.Change_consumer_applied(token,request_id,result) @@ -3904,14 +3926,18 @@ let start List.find_map(fun resident->if resident.Wc.phase=Wc.Open && Wc.matches_scope resident.session owner then Some(owner,host,resident.session) else None)(Wc.residents projection)) |_->None in + let report_asset_coordination owner operation error = + if runtime_owner_is_live !state_ref owner then + run_owned (Some owner) (fun () -> set_state (fun state -> + report_coordination_error state ~operation ~phase:"asset" error)) in asset_demand := (fun ~graph_generation ~name priority assets -> match coordinated_asset_context() with |Some(owner,host,session) when owner.graph_generation=graph_generation-> (match Wc.asset_consumer session ~name with - |Error _->None + |Error error->report_asset_coordination owner "replaceAssetDemand" error;None |Ok consumer-> match Coordination_host.request host session(Wc.Replace_asset_demand(consumer,priority,assets)) with - |Error _->None + |Error error->report_asset_coordination owner "replaceAssetDemand" error;None |Ok ticket-> let media_runtime= !media_runtime_cell and asset_runtime= !asset_runtime_cell in Coordination_host.watch host ticket ~f:(function @@ -3921,7 +3947,10 @@ let start if not(media || assets) then ignore(release_coordinated_consumer(ticket,Some demand):bool) |Answered(Error Revoked_session)|Interrupted Cancelled|Interrupted Worker_stopped->() - |_->if runtime_owner_is_live !state_ref owner then + |failed-> + Option.iter (report_asset_coordination owner "replaceAssetDemand") + (Coordination_host.reply_error failed); + if runtime_owner_is_live !state_ref owner then run_owned(Some owner)(fun()->Effect.of_thunk(fun()-> Journal_media_runtime.demand_backpressured ~ticket media_runtime name; Journal_asset_runtime.demand_backpressured ~ticket asset_runtime name))); @@ -3931,16 +3960,21 @@ let start match coordinated_asset_context() with |Some(owner,host,session) when owner.graph_generation=graph_generation-> (match Coordination_host.request host session(Wc.Retry_asset asset) with - |Error _->false|Ok ticket->Coordination_host.watch host ticket ~f:(fun _->());true) + |Error error->report_asset_coordination owner "retryAsset" error;false + |Ok ticket->Coordination_host.watch host ticket ~f:(fun reply-> + Option.iter (report_asset_coordination owner "retryAsset") + (Coordination_host.reply_error reply));true) |_->false); media_acquire := (fun ticket file -> match coordinated_asset_context() with |Some(owner,host,session)-> (match Coordination_host.request host session(Wc.Acquire_asset_file file) with - |Error _->false + |Error error->report_asset_coordination owner "acquireAssetFile" error;false |Ok acquire-> let runtime= !media_runtime_cell in Coordination_host.watch host acquire ~f:(fun reply-> + Option.iter (report_asset_coordination owner "acquireAssetFile") + (Coordination_host.reply_error reply); let lease=match reply with Wc.Answered(Ok lease)->lease|_->None in if runtime_owner_is_live !state_ref owner then run_owned(Some owner)(fun()-> Effect.bind(Effect.of_thunk(fun()->Journal_media_runtime.receive_file runtime ticket lease)) @@ -3956,7 +3990,7 @@ let start (match Coordination_host.plan_query_when_available host ~owner ~maximum_items:request.maximum_items ~maximum_bytes:Worker_pure_data.maximum_response_bytes request.query with - | Error _->false + | Error error->report_asset_coordination owner "readMedia" error;false | Ok ticket-> Coordination_host.watch_read host ticket ~f:(fun origin reply-> if runtime_owner_is_live !state_ref owner then run_owned (Some owner)(fun()-> @@ -3964,7 +3998,9 @@ let start | Some(session,_),Wc.Answered(Ok value) when Coordination_host.validate_session ~owner host session=Ok()->Ok value | _,Wc.Answered(Error error)->Error error - | _->Error(Wc.Service_failed "The media listing was interrupted or retired") in + | _,(Wc.Not_admitted _ | Wc.Interrupted _) -> + Error(Option.get(Coordination_host.reply_error reply)) + | _->Error(Wc.Service_failed "The media listing belongs to a retired session") in (* The business ID is carried by its private request ticket. *) let request_id=Journal_media_runtime.read_request_id original in apply_media_read ~request_id ~result:(Result.map(fun _->())value) ~f:(fun()-> @@ -3983,7 +4019,11 @@ let start match result,Journal_media_runtime.projection_read_error (!media_runtime_cell) ~root with | Ok(),Some message->Error(Wc.Service_failed message)|_->result)(!media_changes_cell)result in consumed,application,result)) ~f:(fun(consumed,application,result)-> - let flush=match !set_state_ref with None->Effect.ignore|Some set_state->flush_media set_state in + let flush=match !set_state_ref with None->Effect.ignore|Some set_state-> + Effect.bind (match consumed,result with + |true,Error error->set_state(fun state->report_coordination_error state + ~operation:"readOfflineAssets" ~phase:"read" error) + |_->Effect.ignore) ~f:(fun()->flush_media set_state) in Effect.bind flush ~f:(fun()-> Option.iter(fun token->Option.iter(fun(_,host)->Coordination_host.post host (if consumed then Coordination.Change_consumer_applied(token,request_id,result) @@ -3995,7 +4035,7 @@ let start (match Coordination_host.plan_query_when_available host ~owner ~maximum_items:request.maximum_items ~maximum_bytes:Worker_pure_data.maximum_response_bytes request.query with - | Error _->false + | Error error->report_asset_coordination owner "readMedia" error;false | Ok ticket -> let shared = match request.query with | Wc.Journals _ -> true @@ -4010,7 +4050,8 @@ let start | Some(session,_),Wc.Answered(Ok value) when Coordination_host.validate_session ~owner host session=Ok()->Ok value | _,Wc.Answered(Error error)->Error error - | _->Error(Wc.Service_failed "The offline attachment read was interrupted or retired") in + | _,failed->Error(Option.value(Coordination_host.reply_error failed) + ~default:(Wc.Service_failed "The offline attachment read belongs to a retired session.")) in apply_asset_read ~request_id:request.request_id ~result:(Result.map(fun _->())value) ~f:(fun()-> let owned=Journal_asset_runtime.owns_read (!asset_runtime_cell) request.request_id in match value with @@ -4224,6 +4265,8 @@ let start |Confirmation_ticket ticket-> Coordination_host.watch host ticket ~f:(fun reply->f(map_reply(fun confirmation->Wc.Confirmed confirmation)reply)) in + let refuse_coordination error=Journal_graph_transport.Refused + (Coordination_host.diagnostic_error error) in let send_coordinated_read request read = match current_owner(),ensure_coordination() with | Some owner,Ok host -> @@ -4243,16 +4286,13 @@ let start | Answered(Error(Wc.Resync_required message)) when is_changes ->Ok(Journal_graph_runtime.Changes_resync message) | Answered(Error(Wc.Stale_read message))->Ok(Journal_graph_runtime.Read_failed{code=Logseq_db_worker.Error.Stale_read_cursor;message}) | Answered(Error Wc.Response_budget_exceeded)->Ok(Journal_graph_runtime.Read_failed{code=Logseq_db_worker.Error.Response_too_large;message="The read exceeded its response budget."}) - | Answered(Error(Wc.Db_error error))->Ok(Journal_graph_runtime.Read_failed - {code=Logseq_db_worker.Error.code error;message=Logseq_db_worker.Error.message error}) - | Answered(Error(Wc.Service_failed message|Resync_required message))->Error message - | Answered(Error Revoked_session)->Error "The graph execution changed before the read completed." - | Answered(Error _)->Error "The typed graph read could not be validated." - | Not_admitted Full->Error "Worker request queue is full" - | Not_admitted Not_ready->Error "Worker is not ready" - | Not_admitted Stopping->Error "Worker is stopping" - | Not_admitted(Unsupported_request message)->Error message - | Interrupted _->Error "Worker read was interrupted" in + | Answered(Error(Wc.Db_error error))->Ok(Journal_graph_runtime.Read_error error) + | Answered(Error error)->Ok(Journal_graph_runtime.Read_error + (Coordination_host.diagnostic_error error)) + | Not_admitted reason->Ok(Journal_graph_runtime.Read_error + (Coordination_host.admission_diagnostic reason)) + | Interrupted reason->Ok(Journal_graph_runtime.Read_error + (Coordination_host.interruption_diagnostic reason)) in !coordinated_read_finished owner request application origin result) in (match read with @@ -4266,11 +4306,11 @@ let start && Wc.matches_scope resident.session owner then Some resident.session else None)(Wc.residents projection) in (match session with None->Some Journal_graph_transport.Not_ready|Some session-> match Wc.read_budget session ~maximum_items:read.maximum_items ~maximum_bytes:Worker_pure_data.maximum_response_bytes with - | Error _->Some Journal_graph_transport.Not_ready + | Error error->Some(refuse_coordination error) | Ok budget-> observe_projection_query ~request_id:request read.query; match Coordination_host.request host session(Wc.Observe(budget,read.query)) with - | Error _->Some Journal_graph_transport.Not_ready + | Error error->Some(refuse_coordination error) | Ok ticket-> let context:Detail_observation.context={entry_id;request_generation=generation;root=uuid; owner;original=request;maximum_children} in @@ -4292,7 +4332,7 @@ let start observe_projection_query ~request_id:request read.query; watch None ticket (fun value->Journal_graph_runtime.Query_result(read.query,value)); Journal_graph_transport.Accepted - | Error _->Journal_graph_transport.Not_ready) + | Error error->refuse_coordination error) | Confirm_readback read -> (match Coordination_host.reconcile_mutation host ~presentation:read.presentation read.intent with | Ok ticket-> @@ -4305,12 +4345,13 @@ let start |Submission_ticket ticket->watch (Some(Wc.presentation_session read.presentation)) ticket (fun _-> Journal_graph_runtime.Read_failed{code=Logseq_db_worker.Error.Invalid_request;message="The change is awaiting confirmation."})); Journal_graph_transport.Accepted - | Error _->Journal_graph_transport.Not_ready) + | Error error->refuse_coordination error) | Acknowledge read -> (match Coordination_host.request host (Wc.change_batch_session read.batch)(Wc.Acknowledge_changes read.batch) with | Ok ticket->watch(Some(Wc.change_batch_session read.batch))ticket (fun ack->Journal_graph_runtime.Changes_acknowledged ack);Journal_graph_transport.Accepted - | Error _->Journal_graph_transport.Not_ready) + | Error error->refuse_coordination error) ) + | _,Error message->refuse_coordination(Wc.Service_failed message) | _->Journal_graph_transport.Not_ready in let coordinated_mutation_finished=ref(fun (_:Graph_service.timeline_scope) (_:Wc.Uuid.t) (_:(Wc.mutation_progress,string)result)->()) in @@ -4330,12 +4371,13 @@ let start | Wc.Answered(Ok(Wc.Committed_pending_presentation _))-> (match Coordination_host.reconcile_mutation host ~presentation intent with | Ok ticket->watch ticket - | Error _-> !coordinated_mutation_finished owner request(Error "The change is awaiting a current presentation.")) + | Error error-> !coordinated_mutation_finished owner request(Error(coordinated_error_message error))) | Answered(Ok progress)-> !coordinated_mutation_finished owner request(Ok progress) - | _-> !coordinated_mutation_finished owner request(Error "The change could not be confirmed.")) in + | failed-> !coordinated_mutation_finished owner request(Error + (Coordination_host.reply_error_message ~fallback:"The change could not be confirmed." failed))) in let dispatch ()=match Coordination_host.reconcile_mutation host ~presentation intent with | Ok ticket->watch ticket;Journal_graph_transport.Accepted - | Error _->Journal_graph_transport.Not_ready in + | Error error->refuse_coordination error in let snapshot=match Journal_graph_runtime.capture_snapshot (!graph_runtime_cell) request with | Some(command,chain)->Some(`Capture(command,chain)) | None->Option.map(fun(command,chain)->`Child(command,chain)) @@ -4362,11 +4404,12 @@ let start (match Option.bind(Coordination.projection(Coordination_host.view host))Wc.selected with | Some presentation when Wc.matches_scope(Wc.presentation_session presentation)owner -> (match Coordination_host.settle_applied_mutation host ~presentation ~applied_presentation intent with - | Error _->Journal_graph_transport.Not_ready + | Error error->refuse_coordination error | Ok(Coordination.Acknowledgement ticket)-> Coordination_host.watch host ticket ~f:(fun reply-> let result=match reply with Wc.Answered(Ok())->Ok() - | _->Error "The child result is applied, but its acknowledgement could not finish." in + | failed->Error(Coordination_host.reply_error_message + ~fallback:"The child result is applied, but its acknowledgement could not finish." failed) in !coordinated_child_ack_finished owner request presentation result); Journal_graph_transport.Accepted | Ok(Coordination.Reconciliation ticket)-> @@ -4376,9 +4419,10 @@ let start | Wc.Answered(Ok(Wc.Committed_pending_presentation _)) when bound -> (match Coordination_host.reconcile_mutation host ~presentation intent with | Ok ticket->watch ticket - | Error _-> !coordinated_mutation_finished owner request(Error "The child awaits a current presentation.")) + | Error error-> !coordinated_mutation_finished owner request(Error(coordinated_error_message error))) | Answered(Ok progress) when bound -> !coordinated_mutation_finished owner request(Ok progress) - | _-> !coordinated_mutation_finished owner request(Error "The child could not be reconciled.")) in + | failed-> !coordinated_mutation_finished owner request(Error + (Coordination_host.reply_error_message ~fallback:"The child could not be reconciled." failed))) in watch ticket;Journal_graph_transport.Accepted) | _->Journal_graph_transport.Not_ready) | _->Journal_graph_transport.Not_ready in @@ -4486,9 +4530,9 @@ let start | Wc.Answered(Ok(Wc.Committed_pending_presentation _))-> (match Coordination_host.reconcile_mutation host ~presentation intent with | Ok ticket->watch ticket - | Error _->finish(Error "The change is awaiting a current presentation.")) + | Error error->finish(Error(coordinated_error_message error))) | Wc.Answered(Ok progress)->finish(Ok progress) - | _->finish(Error "The change could not be confirmed.")) in + | failed->finish(Error(Coordination_host.reply_error_message ~fallback:"The change could not be confirmed." failed))) in watch ticket))) in mutation_acknowledgements_changed:=(fun()-> match !coordination_host,timeline_scope !state_ref with @@ -4725,7 +4769,7 @@ let start ~request_id:request query value; Journal_asset_runtime.observe_query_result (!asset_runtime_cell) ~request_id:request query value - | Read_failed _ | Changes_acknowledged _ | Changes_resync _ | Changes_retired -> + | Read_failed _ | Read_error _ | Changes_acknowledged _ | Changes_resync _ | Changes_retired -> forget_projection_request request); asset_projection_error := (if asset_error_before=None then Journal_asset_runtime.projection_read_error (!asset_runtime_cell) @@ -5417,18 +5461,21 @@ let start (match session with |None->Error "The destination is not ready for imports." |Some session->(match Wc.asset_import_intent session source with - |Error _->Error "The attachment source or immutable identity is invalid." + |Error error->Error(coordinated_error_message error) |Ok intent->(match Coordination_host.request host session(Wc.Import intent) with - |Error _->Error "Import is temporarily unavailable." + |Error error->Error(coordinated_error_message error) |Ok ticket-> Coordination_host.watch host ticket ~f:(fun reply-> let result=match reply with |Wc.Answered(Ok(Wc.Imported imported))->Ok imported - |Answered(Error(Wc.Service_failed message|Invalid_input message))->Error message - |_->let _,operation,_=pending in - Error(if Hashtbl.mem (!capture_staged_items_cell) operation - then "Import was interrupted. Your journal entry and attachment input are retained." - else "Import was interrupted. Select the file again.") in + |failed->let _,operation,_=pending in + let recovery=if Hashtbl.mem (!capture_staged_items_cell) operation + then "Import outcome is unknown. Your journal entry and attachment input are retained." + else "Import outcome is unknown. Select the file again." in + let cause=Coordination_host.reply_error_message ~fallback:recovery failed in + Error(match failed with + | Wc.Interrupted _ when cause<>recovery -> cause ^ ". " ^ recovery + | _ -> cause) in if runtime_owner_is_live !state_ref owner then run_owned(Some owner)(fun()->complete_import(Some pending)result) else Result.iter(fun imported->Option.iter(fun lease->ignore(release_coordinated_file lease:bool))(Wc.import_preview imported))result); @@ -8369,14 +8416,14 @@ let create latest_patch := ""; Printexc.record_backtrace true; (match decode_config (Bytes.of_string payload) with - | Error _ -> failwith "application.decodeStartup.failed" + | Error source -> raise (Journal_bridge.initialization_failure Decode_startup source) | Ok config -> let runtime_epoch = Journal_worker_ids.Runtime.Epoch.of_int64 (Int64.of_float (Unix.gettimeofday () *. 1e6)) in (match Journal_worker_runtime.start ~runtime_epoch service config with - | Error _ -> failwith "application.startWorker.failed" + | Error source -> raise (Journal_bridge.initialization_failure Start_worker source) | Ok client -> (try ignore @@ -8389,12 +8436,13 @@ let create ~host_code); Option.iter (fun observe -> observe client) on_client with - | _ -> + | error -> + let backtrace=Printexc.get_raw_backtrace () in (match !current_app with | Some context when context.client == client -> dispose_context context | _ -> ()); Journal_worker_runtime.stop client; - failwith "application.initialize.failed"))); + Printexc.raise_with_backtrace (Journal_bridge.initialization_exception error) backtrace))); !latest_patch in let dispatch event = @@ -8420,7 +8468,10 @@ let create (fun () -> (match Lui_runtime.extension_identifier (Lui_app.runtime app) node with | Some identifier -> - let values = decode_extension_values values in + (match decode_extension_values values with + | Error source->send_action(Update(fun state-> + report_decode_error state ~operation:"decodeExtensionEvent" source,Effect.ignore)) + | Ok values-> let report operation = send_action (Update (fun state -> report_ui_error state ~operation ~phase:"native", Effect.ignore)) in (match name, Lui_protocol.String_map.find_opt "id" values, @@ -8436,7 +8487,7 @@ let create | "event", _, _ -> report "decodeNativeEvent" | _ -> ignore (Lui_app.dispatch_event app - (Lui_protocol.ExtensionEvent (node, identifier, name, values)))) + (Lui_protocol.ExtensionEvent (node, identifier, name, values))))) | None -> ()); ignore (Lui_app.flush app)) | None -> ()); diff --git a/app/impl/worker_coordination/worker_effect_authority.mli b/app/impl/worker_coordination/worker_effect_authority.mli index cec48c2d..11b55e5d 100644 --- a/app/impl/worker_coordination/worker_effect_authority.mli +++ b/app/impl/worker_coordination/worker_effect_authority.mli @@ -439,7 +439,7 @@ val maximum_mutation_recovery_bytes : int type interruption = | Cancelled - | Transport_lost + | Transport_lost of error | Worker_stopped type 'a reply = diff --git a/app/impl/worker_coordination/worker_effect_runner.ml b/app/impl/worker_coordination/worker_effect_runner.ml index dbeb1a95..80002149 100644 --- a/app/impl/worker_coordination/worker_effect_runner.ml +++ b/app/impl/worker_coordination/worker_effect_runner.ml @@ -212,6 +212,15 @@ let submit t = function | Schedule_drain -> t.post Core.Drain ;; +let retirement_result operation decode = function + | Mailbox.Completed response -> decode response + | Failed message -> Error (Worker.Service_failed (operation ^ ": " ^ message)) + | Cancelled -> + Error (Worker.Service_failed (operation ^ ": cancelled before retirement acknowledgement")) + | Shutdown -> + Error (Worker.Service_failed (operation ^ ": worker stopped before retirement acknowledgement")) +;; + let on_event t event = let runtime, worker = match event with @@ -288,7 +297,10 @@ let on_event t event = Worker.Answered (Error e)) | Failed message -> (match request with - | Worker.Import _ -> Worker.Interrupted Worker.Transport_lost + | Worker.Import _ -> + let cause = Worker.Service_failed ("Import: " ^ message) in + error := Some cause; + Worker.Interrupted (Worker.Transport_lost cause) | _ -> Worker.Answered (Error (Worker.Service_failed message))) | Cancelled -> Worker.Interrupted Worker.Cancelled | Shutdown -> Worker.Interrupted Worker.Worker_stopped @@ -301,19 +313,14 @@ let on_event t event = | Unobserve (issuer, session, id) -> let module I = (val issuer : Worker_effect_authority.ISSUER) in let result = - match outcome with - | Completed response -> - I.observation_registration_released session ~observation_id:id response - | _ -> - Error - (Worker.Service_failed - "Observation release did not acknowledge physical retirement") + retirement_result "Observation release" + (I.observation_registration_released session ~observation_id:id) outcome in t.post (Core.Observation_release_completed (session, id, result)) |Finalize_observation(issuer,session,id)-> let module I=(val issuer:Worker_effect_authority.ISSUER) in - let result=match outcome with Completed response->I.observation_registration_finalized session ~observation_id:id response - |_->Error(Worker.Service_failed "Observation finalization was not acknowledged") in + let result=retirement_result "Observation finalization" + (I.observation_registration_finalized session ~observation_id:id) outcome in t.post(Core.Observation_finalize_completed(session,id,result)) |Release_demand(issuer,ticket,registration)-> let module I=(val issuer:Worker_effect_authority.ISSUER) in @@ -357,14 +364,7 @@ let on_event t event = |Ok receipt->Worker.Safe_to_proceed receipt|Error error->Worker.Handoff_failed(change,error))) | Unregister issuer -> let module I = (val issuer : Worker_effect_authority.ISSUER) in - let result = - match outcome with - | Completed response -> I.binding_released response - | _ -> - Error - (Worker.Service_failed - "Unregister did not acknowledge resource retirement") - in + let result = retirement_result "Binding unregister" I.binding_released outcome in t.post (Core.Binding_release_completed (I.binding, result))); Consumed) | Push { payload = Service.Graph_pushes _ as payload; push_sequence; _ } -> @@ -410,7 +410,7 @@ let on_event t event = | None->() | Some resident->(match I.asset_notice resident.session data with | Ok notice->t.post(Core.Asset_received(notice,push_sequence)) - | Error _->()))(Service.asset_notices payload)) + | Error cause->error := Some cause))(Service.asset_notices payload)) (Core.projection view))t.latest; Broadcast | Push _ -> Broadcast diff --git a/app/impl/worker_coordination/worker_pure_api.ml b/app/impl/worker_coordination/worker_pure_api.ml index ccdaed79..7688478b 100644 --- a/app/impl/worker_coordination/worker_pure_api.ml +++ b/app/impl/worker_coordination/worker_pure_api.ml @@ -121,7 +121,7 @@ type admission_error = type interruption = | Cancelled - | Transport_lost + | Transport_lost of error | Worker_stopped type 'a reply = diff --git a/app/impl/worker_coordination/worker_pure_coordinator.ml b/app/impl/worker_coordination/worker_pure_coordinator.ml index d9ca1d16..3522e7e4 100644 --- a/app/impl/worker_coordination/worker_pure_coordinator.ml +++ b/app/impl/worker_coordination/worker_pure_coordinator.ml @@ -2557,6 +2557,13 @@ let plan_host_action_attempt (state:t) receipt = host_consume_early=None;host_consume_waiting=false},Consume_handoff(receipt,attempt)) ;; +let cleanup_admission_error operation = function + | Worker.Full -> None + | Not_ready -> Some (Worker.Service_failed (operation ^ ": worker not ready")) + | Stopping -> Some (Worker.Service_failed (operation ^ ": worker stopping")) + | Unsupported_request reason -> Some (Worker.Unsupported_operation (operation ^ ": " ^ reason)) +;; + let step_base state event = match event with | Import_source_proof_returned proof -> @@ -3002,11 +3009,7 @@ let step_base state event = | Some (Observing value) when Worker.equal_session session value.obs_session && value.obs_release_inflight && not value.obs_release_done -> - let error = - match admission with - | Worker.Full -> None - | _ -> Some (Worker.Service_failed "Observation cleanup not admitted") - in + let error = cleanup_admission_error "Observation cleanup" admission in changed { state with facts = @@ -3058,7 +3061,7 @@ let step_base state event = (match Observation_ids.find_opt id state.facts.observations with |Some(Observing value) when value.obs_release_done && value.obs_release_inflight && Worker.equal_session session value.obs_session-> - let error=if admission=Worker.Full then None else Some(Worker.Service_failed "Observation finalization not admitted") in + let error=cleanup_admission_error "Observation finalization" admission in let observations=Observation_ids.add id (Observing{value with obs_release_inflight=false; obs_cleanup_waiting=admission=Worker.Full;obs_cleanup_error=error})state.facts.observations in changed {state with facts={state.facts with observations}} [] diff --git a/app/journal_asset_import.ml b/app/journal_asset_import.ml index 7065335e..0412e018 100644 --- a/app/journal_asset_import.ml +++ b/app/journal_asset_import.ml @@ -165,11 +165,16 @@ let extension = (* Staged picks live under the host temp directory as [journal-import-*]; delete only files matching that contract so a user path is never removed. *) -let discard_staged_file (staged : staged) = +let discard_staged_file ?(on_failure=fun message-> + try prerr_endline ("journal: operation=discardStagedFile " ^ message) with _->()) (staged : staged) = if String.starts_with ~prefix:"journal-import-" (Filename.basename staged.source_file) then ( - try Sys.remove staged.source_file with - | _ -> ()) + try Unix.unlink staged.source_file with + | Unix.Unix_error(Unix.ENOENT,_,_)->() + | error-> + let module D=Logseq_db_types.Failure_diagnostic in + let detail=D.of_source ~operation:"assetRelease" ~reason:"cleanupFailed" (Printexc.to_string error) in + on_failure(D.encode detail)) ;; let is_dismissal payload = diff --git a/app/journal_asset_import.mli b/app/journal_asset_import.mli index 897c4532..7097f91f 100644 --- a/app/journal_asset_import.mli +++ b/app/journal_asset_import.mli @@ -50,8 +50,10 @@ type event = val decode_event : string -> (event, string) result (** Best-effort removal of a staged temp copy created for this pick; a no-op - for paths outside the staged temp-file naming contract. *) -val discard_staged_file : staged -> unit + for paths outside the staged temp-file naming contract or already absent files. + Other failures reach [on_failure] as a bounded diagnostic carrier; the default + writes the same safe cause to the local diagnostic stream. *) +val discard_staged_file : ?on_failure:(string -> unit) -> staged -> unit val is_dismissal : string -> bool diff --git a/app/journal_bridge.ml b/app/journal_bridge.ml index ddfa48c8..95eba473 100644 --- a/app/journal_bridge.ml +++ b/app/journal_bridge.ml @@ -85,7 +85,77 @@ let platform_failure payload = (hooks ()).platform_failure payload let dispose () = (hooks ()).dispose () let root_node () = (hooks ()).root_node () +type initialization_stage = Decode_startup | Start_worker | Initialize + +(* Stable bounded codes shared with JournalRuntime; zero is an explicit unknown. + No strings from producers or exceptions are retained by the exception. *) +exception Initialization_failure of initialization_stage * int + +let initialization_cause = function + | "startup payload is truncated" -> 1 + | "startup payload exceeds 1 MiB" -> 2 + | "invalid startup magic" -> 3 + | "startup configuration length is invalid" -> 4 + | "startup payload has trailing or missing bytes" -> 5 + | "startup configuration must be valid UTF-8 JSON" -> 6 + | "invalid startup configuration: application_support_directory must be a bounded canonical absolute path" -> 7 + | "invalid startup configuration: response_budget_bytes is outside the protocol bounds" -> 8 + | "invalid startup configuration: default_page_size is outside the protocol bounds" -> 9 + | "invalid startup configuration: sync base URL must be one HTTPS origin without credentials or fragments" -> 10 + | "worker.start.failure" -> 12 + | "worker.start.invalidArgument" -> 13 + | "worker.start.systemError" -> 14 + | "worker.start.permissionDenied" -> 15 + | "worker.start.missingFile" -> 16 + | "worker.start.noSpace" -> 17 + | "worker.start.busy" -> 18 + | "worker.start.io" -> 19 + | "worker.start.unixError" -> 20 + | "worker.start.outOfMemory" -> 21 + | "worker.start.stackOverflow" -> 22 + | "worker.start.endOfFile" -> 23 + | "worker.start.matchFailure" -> 24 + | "worker.start.assertFailure" -> 25 + | "worker.start.notFound" -> 26 + | "application.mount.failed" -> 27 + | "Worker Domain subsystem is stopped" | "Worker Domain has stopped" -> 28 + | "Worker Domain subsystem is terminal" -> 29 + | "Worker Domain session is already attached" -> 30 + | "Worker Domain did not start" -> 31 + | _ -> 0 +;; + +let initialization_failure stage source = + Initialization_failure (stage, initialization_cause source) +;; + +let initialization_exception = function + | Initialization_failure _ as error -> error + | error -> + let cause = match error with + | Failure message when message = "application.mount.failed" -> 27 + | Failure _ -> 12 + | Invalid_argument _ -> 13 + | Sys_error _ -> 14 + | Unix.Unix_error ((Unix.EACCES | Unix.EPERM), _, _) -> 15 + | Unix.Unix_error (Unix.ENOENT, _, _) -> 16 + | Unix.Unix_error (Unix.ENOSPC, _, _) -> 17 + | Unix.Unix_error ((Unix.EBUSY | Unix.EAGAIN), _, _) -> 18 + | Unix.Unix_error (Unix.EIO, _, _) -> 19 + | Unix.Unix_error _ -> 20 + | Out_of_memory -> 21 + | Stack_overflow -> 22 + | End_of_file -> 23 + | Match_failure _ -> 24 + | Assert_failure _ -> 25 + | Not_found -> 26 + | _ -> 0 + in Initialization_failure (Initialize, cause) +;; + let classify_exception = function + | Initialization_failure (stage, cause) -> + (match stage with Decode_startup -> 500 | Start_worker -> 600 | Initialize -> 700) + cause | Failure message when message = "application.decodeStartup.failed" -> 5 | Failure message when message = "application.startWorker.failed" -> 6 | Failure message when message = "application.mount.failed" -> 7 diff --git a/app/journal_bridge.mli b/app/journal_bridge.mli index 64f8958b..aec0ee3f 100644 --- a/app/journal_bridge.mli +++ b/app/journal_bridge.mli @@ -61,3 +61,12 @@ val platform_failure_message : string -> string (** Recognized native diagnostic family and finite subcause. Unrecognized suffixes use a fixed unknown reason; arbitrary payloads are never echoed. *) val native_diagnostic : string -> (string * string) option + + +(** Startup failures retain a finite operation and cause without exception + arguments, paths, configuration values or credentials. *) +type initialization_stage = Decode_startup | Start_worker | Initialize +val initialization_failure : initialization_stage -> string -> exn +(** Preserve an existing typed startup failure, or classify an exception by its + constructor and a bounded Unix error category. *) +val initialization_exception : exn -> exn diff --git a/app/journal_detail.ml b/app/journal_detail.ml index 2056c752..bdc410dc 100644 --- a/app/journal_detail.ml +++ b/app/journal_detail.ml @@ -1042,7 +1042,9 @@ module Pure_observation = struct | Not_admitted Stopping -> Some (Worker.Service_failed "Worker is stopping.") | Not_admitted (Unsupported_request message) -> Some (Worker.Unsupported_operation message) | Answered (Error error) -> Some error - | Interrupted _ -> Some Worker.Revoked_session + | Interrupted (Transport_lost error) -> Some error + | Interrupted Cancelled -> Some(Worker.Service_failed "Observation request was cancelled.") + | Interrupted Worker_stopped -> Some(Worker.Service_failed "Worker stopped during observation.") | Answered (Ok _) -> None ;; diff --git a/app/journal_graph_favorites_plan.ml b/app/journal_graph_favorites_plan.ml index 31ee1460..9039b097 100644 --- a/app/journal_graph_favorites_plan.ml +++ b/app/journal_graph_favorites_plan.ml @@ -27,6 +27,7 @@ type event = | Favorites_loaded of Journal_graph_request.favorites_request * Journal_worker_coordination.Worker_api.favorites | Favorites_failed of Journal_graph_request.favorites_request * bool * string | Favorites_invalidated + | Diagnostic of { operation : string; request_id : Graph.Uuid.t; error : Error.t } | Rejected of rejection type output = { requests : Graph.Uuid.t Rrbvec.t @@ -509,7 +510,9 @@ let fail t operation ~request_id ~code ~message ~error = let rejected () = responses [ Rejected (Worker_rejection { operation = operation_name operation; request_id; error }) ] in match operation with - | List_favorites request -> responses [Favorites_failed (request, code = Error.Stale_read_cursor, message)] + | List_favorites request -> + responses [Favorites_failed (request, code = Error.Stale_read_cursor, message); + Diagnostic {operation=operation_name operation;request_id;error}] | Favorite_dependency {epoch;uuid;_} -> if epoch <> t.favorite_epoch then empty else complete_favorite_dependency t uuid (rejected ()) | Favorite_memberships {epoch;page;_} -> diff --git a/app/journal_graph_favorites_plan.mli b/app/journal_graph_favorites_plan.mli index 3cc09e38..5facf9d0 100644 --- a/app/journal_graph_favorites_plan.mli +++ b/app/journal_graph_favorites_plan.mli @@ -11,6 +11,7 @@ type event = | Favorites_loaded of Journal_graph_request.favorites_request * Journal_worker_coordination.Worker_api.favorites | Favorites_failed of Journal_graph_request.favorites_request * bool * string | Favorites_invalidated + | Diagnostic of { operation : string; request_id : Graph.Uuid.t; error : Error.t } | Rejected of rejection type output = { requests : Graph.Uuid.t Rrbvec.t diff --git a/app/journal_graph_feed_plan.ml b/app/journal_graph_feed_plan.ml index b42c8cdf..f398580b 100644 --- a/app/journal_graph_feed_plan.ml +++ b/app/journal_graph_feed_plan.ml @@ -262,8 +262,8 @@ let continue_root_batch t ~ports owner page cursor limit = then cursor_failure batch "The retained root page changed. Refresh Timeline." else ( match ports.validate_session cursor.session with - | Error _ -> - cursor_failure batch "The retained root execution was revoked. Refresh Timeline." + | Error error -> + cursor_failure batch (Journal_worker_coordination_host.error_message error) | Ok () -> (match Worker.validate_request @@ -277,8 +277,8 @@ let continue_root_batch t ~ports owner page cursor limit = ; after = cursor.after } )) with - | Error _ -> - cursor_failure batch "The retained root query no longer matches its budget." + | Error error -> + cursor_failure batch (Journal_worker_coordination_host.error_message error) | Ok () when not (Rrbvec.is_empty cursor.remaining) -> { empty with cached = Some batch } | Ok () when cursor.after <> None -> requests [ plan_root_batch ~ports batch ] @@ -815,8 +815,8 @@ let resume_cached t ~ports batch = then superseded_root_batch t ~ports batch else ( match ports.validate_session cursor.Projection.session with - | Error _ -> - cursor_failure batch "The retained root execution was revoked. Refresh Timeline." + | Error error -> + cursor_failure batch (Journal_worker_coordination_host.error_message error) | Ok () -> receive_root_tree t diff --git a/app/journal_graph_mutation_confirmation.ml b/app/journal_graph_mutation_confirmation.ml index 25da7def..2c62a540 100644 --- a/app/journal_graph_mutation_confirmation.ml +++ b/app/journal_graph_mutation_confirmation.ml @@ -301,7 +301,7 @@ module Pure_insertion = struct then Error "The atomic Capture children do not match the admitted payload." else Result.bind(Graph.Uuid.of_string command.mutation_id)(fun mutation_id-> match Worker.journal_day day with - |Error _->Error "Capture requires its admitted day and atomic Append payload." + |Error error->Error(Journal_worker_coordination_host.error_message error) |Ok journal_day->Result.map(fun(t,action)->{t with capture=Some command},action) (start_value ~command:(capture_command command) ~page:None ~resolved_status ~append_day:(Some day) (Append{mutation_id;day=journal_day;root}))) diff --git a/app/journal_graph_runtime.ml b/app/journal_graph_runtime.ml index 543c9140..b5f71444 100644 --- a/app/journal_graph_runtime.ml +++ b/app/journal_graph_runtime.ml @@ -12,6 +12,7 @@ module Coordination = Journal_worker_coordination.Worker_pure_coordinator type worker_read_result = | Query_result : 'a Worker.query * 'a -> worker_read_result | Read_failed of { code : Logseq_db_worker.Error.code; message : string } + | Read_error of Logseq_db_worker.Error.t | Changes_resync of string | Changes_retired | Changes_acknowledged of Worker.change_ack @@ -160,6 +161,7 @@ type payload = ; block_id : string ; failure : failure_source } + | Diagnostic of worker_failure | Rejected of failure_source type response = { payload : payload } @@ -523,6 +525,8 @@ let favorites_output (value : Favorites.output) = | Favorites_failed (request, stale, message) -> response (Favorites_failed (request, stale, message)) | Favorites_invalidated -> response Favorites_invalidated | Rejected (Projection_rejection message) -> response (Rejected (Projection_failure message)) + | Diagnostic { operation; request_id; error } -> + response (Diagnostic { operation; request_id; error }) | Rejected (Worker_rejection { operation; request_id; error }) -> response (Rejected (Worker_failure { operation; request_id; error }))) value.responses) @@ -1805,10 +1809,10 @@ let operation_name = function let worker_error code request_id message = Error.create ~code ~message ~details:[] |> Result.fold ~ok:Fun.id ~error:(fun _ -> - Error.create - ~code:Unsupported_semantics - ~message:"The Worker request failed." - ~details:[] + let module D=Logseq_db_types.Failure_diagnostic in + let detail=D.of_source ~operation:"workerOperation" ~reason:"invalidContent" message in + Error.create ~code ~message:(D.message detail) + ~details:[Error.{name=D.detail_name;value=Detail_string(D.encode detail)}] |> Result.get_ok) |> fun error -> request_id, error ;; @@ -1816,19 +1820,22 @@ let worker_error code request_id message = let failure_output ~source_epoch ?(code = Error.Unsupported_semantics) + ?error t operation request_id message = - let request_id, error = worker_error code request_id message in + let request_id, error = match error with + | Some error -> request_id,error + | None -> worker_error code request_id message in let worker_failure = { operation = operation_name operation; request_id; error } in match operation with | Task_catalog { epoch; _ } when epoch <> t.task_catalog_epoch -> empty | Task_catalog _ -> t.task_catalog_loading <- false; Hashtbl.clear t.task_catalog_deletions; - responses [ task_catalog_response ~error:message t ] + responses [ task_catalog_response ~error:message t; response(Diagnostic worker_failure) ] | Task_definitions ids -> let current = ref false in List.iter @@ -1839,7 +1846,8 @@ let failure_output current := true; Hashtbl.remove t.task_lookup_epochs id)) ids; - if !current then responses [ task_catalog_response ~error:message t ] else empty + if !current then responses [ task_catalog_response ~error:message t; + response(Diagnostic worker_failure) ] else empty | Favorite_read value -> Favorites.fail t.favorites value ~request_id ~code ~message ~error |> favorites_output | Feed_read operation -> @@ -1850,7 +1858,8 @@ let failure_output | Mutation_refresh { page; refresh = Updated { block_id } } when code = Error.Conflict -> request_target_summary t page (Update_conflict_refresh { block_id }) | Graph_info -> responses [ response (Open_failed worker_failure) ] - | Admission_info request -> responses [ response (Admission_unavailable request) ] + | Admission_info request -> responses [ response (Admission_unavailable request); + response(Diagnostic worker_failure) ] | Pull_changes _ | Acknowledge_changes _ -> responses [ response (Rejected (Worker_failure worker_failure)) ] | Detail_block { generation; block_id; _ } -> @@ -1875,11 +1884,15 @@ let failure_output ; failure = Worker_failure worker_failure }) ] - | Child_chain _ | Capture_chain _ -> child_step t request_id(fun chain->Child.interrupted chain ~index:0 ~round:1L message) + | Child_chain _ | Capture_chain _ -> + let output=child_step t request_id(fun chain->Child.interrupted chain ~index:0 ~round:1L message) in + {output with responses=response(Diagnostic worker_failure)::output.responses} | Child_write{root;index;round}|Child_ack{root;index;round;_}-> - child_step t root(fun chain->Child.interrupted chain ~index ~round message) + let output=child_step t root(fun chain->Child.interrupted chain ~index ~round message) in + {output with responses=response(Diagnostic worker_failure)::output.responses} | Child_canonical{root;round}|Child_chain_parent{root;round;_}-> - child_step t root(fun chain->Child.read_failed chain ~round message) + let output=child_step t root(fun chain->Child.read_failed chain ~round message) in + {output with responses=response(Diagnostic worker_failure)::output.responses} | Child_insert (command, _) -> responses [ response @@ -1890,8 +1903,9 @@ let failure_output }) ] | Reference_read { block_id; depth; _ } -> - Reference.read_failed t.references ~ports:(reference_ports t) ~source_epoch ~block_id ~depth - |> reference_output + let output=Reference.read_failed t.references ~ports:(reference_ports t) ~source_epoch ~block_id ~depth + |> reference_output in + {output with responses=response(Diagnostic worker_failure)::output.responses} | Changed_block { block_id; _ } | Changed_detail_block { block_id; _ } | Changed_point_context {block_id;_} -> let invalidation = Reference.shared_read_failed t.references ~ports:(reference_ports t) @@ -1969,6 +1983,8 @@ let receive_result t ~origin ~request_id result = Feed.receive t.feed ~ports:(feed_ports t) ?origin operation query value | Feed_read operation,Read_failed{code;_} -> Feed.failed t.feed ~ports:(feed_ports t) operation ~code + | Feed_read operation,Read_error error -> + Feed.failed t.feed ~ports:(feed_ports t) operation ~code:(Error.code error) | _ -> None) with | Some planned -> feed_output t planned | _ -> (match result with @@ -1978,15 +1994,19 @@ let receive_result t ~origin ~request_id result = Favorites.receive_list t.favorites ~ports:(favorites_ports t) read result |> favorites_output |> background_demands t | _ -> failure_output t operation request_id "Unexpected favorites response.") - | Read_failed { code; message } -> + | (Read_failed _ | Read_error _) as failed -> + let code,message,error=match failed with + | Read_failed{code;message}->code,message,None + | Read_error error->Error.code error,Error.message error,Some error + | _->assert false in (match operation with | Target_summary summary when code = Error.Stale_read_cursor -> (match Confirmation.retry_stale t.confirmations ~ports:(confirmation_ports t) summary with | Some planned -> confirmation_planning planned - | None -> failure_output ~code t operation request_id message) + | None -> failure_output ~code ?error t operation request_id message) | Delete_mutation command when code = Error.Conflict -> request_delete_summary t command.block_id - | _ -> failure_output ~code t operation request_id message) + | _ -> failure_output ~code ?error t operation request_id message) | Changes_resync reason -> failure_output t operation request_id reason | Changes_retired -> empty | Query_result (Worker.Graph_info, value) -> @@ -2845,6 +2865,7 @@ let receive_delivery t ~origin ~request_id result = t.on_change(Coordination.Change_consumer_expanded(token,request_id,!children)); let error=match result with | Read_failed{code;message}->Some(Worker.Service_failed(Error.code_string code ^ ": " ^ message)) + | Read_error error->Some(Worker.Db_error error) | Changes_resync reason->Some(Worker.Resync_required reason) | _->List.find_map response_application_error output.responses in Option.iter(fun error->t.on_change(Coordination.Change_consumer_applied(token,request_id,Error error))) error; @@ -3049,10 +3070,15 @@ let receive_mutation t (request:Graph.Uuid.t) progress = (* The known rejection is terminal evidence; only this branch may report that the choice did not apply. *) Hashtbl.replace t.pending key {pending with durable=None}; - let code,message=match error with Worker.Db_error error->Error.code error,Error.message error - | Service_failed message->Error.Invalid_request,message - | _->Error.Invalid_request,"The change was rejected." in - let output=receive_worker_read t ~request_id:request (Read_failed{code;message}) in + let failure=match error with + | Worker.Db_error error->Read_error error + | Service_failed message | Invalid_input message | Protocol_mismatch message + | Unsupported_operation message | Stale_read message | Resync_required message-> + Read_failed{code=Error.Invalid_request;message} + | Revoked_session->Read_failed{code=Error.Closed_session;message="The mutation session was revoked."} + | Host_change_in_progress->Read_failed{code=Error.Closed_session;message="The graph is closing; the mutation outcome is retained."} + | Response_budget_exceeded->Read_failed{code=Error.Response_too_large;message="The mutation result exceeded its response budget."} in + let output=receive_worker_read t ~request_id:request failure in Hashtbl.replace t.pending key {pending with durable=Some durable}; List.iter(fun (request:Graph.Uuid.t)->let key=Graph.Uuid.to_string request in Option.iter(fun pending->Hashtbl.replace t.pending key {pending with durable=Some durable}) @@ -3092,7 +3118,7 @@ let child_projection_applied t ~root ~round ~presentation = dispatch_output t (child_step t root(fun chain->Child.applied chain ~round presentation)) ;; -let fail_request t (request : Graph.Uuid.t) ~message = +let fail_request ?error t (request : Graph.Uuid.t) ~message = match Hashtbl.find_opt t.pending(Graph.Uuid.to_string request) with | Some{operation=Capture_append command;_}-> Hashtbl.remove t.pending(Graph.Uuid.to_string request); @@ -3101,7 +3127,9 @@ let fail_request t (request : Graph.Uuid.t) ~message = Hashtbl.remove t.pending(Graph.Uuid.to_string request); responses[response(Child_not_admitted{command;message})] | Some{operation=(Child_chain _|Capture_chain _|Child_write _|Child_ack _);_}->mutation_waiting t request message - | _->receive_worker_read t ~request_id:request (Read_failed{code=Error.Closed_session;message}) + | _->receive_worker_read t ~request_id:request + (match error with Some error->Read_error error + | None->Read_failed{code=Error.Closed_session;message}) ;; let abandon t (request : Graph.Uuid.t) = diff --git a/app/journal_graph_runtime.mli b/app/journal_graph_runtime.mli index 98bbb5a0..129666cc 100644 --- a/app/journal_graph_runtime.mli +++ b/app/journal_graph_runtime.mli @@ -125,6 +125,7 @@ type payload = ; block_id : string ; failure : failure_source } + | Diagnostic of worker_failure | Rejected of failure_source type response = { payload : payload } @@ -144,7 +145,7 @@ val set_calendar : t -> Journal_calendar.t -> unit val start : t -> Logseq_db_types.Graph_types.Uuid.t val submit : ?capture_chain:Journal_graph_mutation_confirmation.Pure_insertion.t -> ?child_chain:Journal_graph_mutation_confirmation.Pure_insertion.t -> t -> Journal_graph_request.t -> output -val fail_request : t -> Logseq_db_types.Graph_types.Uuid.t -> message:string -> output +val fail_request : ?error:Logseq_db_worker.Error.t -> t -> Logseq_db_types.Graph_types.Uuid.t -> message:string -> output val abandon : t -> Logseq_db_types.Graph_types.Uuid.t -> unit val reconcile_signal @@ -159,6 +160,7 @@ module Worker = Journal_worker_coordination.Worker_api type worker_read_result = | Query_result : 'a Worker.query * 'a -> worker_read_result | Read_failed of { code : Logseq_db_worker.Error.code; message : string } + | Read_error of Logseq_db_worker.Error.t | Changes_resync of string | Changes_retired | Changes_acknowledged of Worker.change_ack diff --git a/app/journal_graph_transport.ml b/app/journal_graph_transport.ml index 5a203608..45a2da82 100644 --- a/app/journal_graph_transport.ml +++ b/app/journal_graph_transport.ml @@ -3,6 +3,7 @@ type send_result = | Full | Not_ready | Stopping + | Refused of Logseq_db_worker.Error.t type delivery = { responses : Journal_graph_runtime.response list @@ -14,25 +15,28 @@ let error_message = function | Full -> Some "Worker request queue is full" | Not_ready -> Some "Worker is not ready" | Stopping -> Some "Worker is stopping" + | Refused error -> Some(Logseq_db_worker.Error.message error) ;; let deliver ~runtime ~send (output : Journal_graph_runtime.output) = let responses = ref (Rrbvec.of_list output.responses) in - let retire_unsent message requests = + let retire_unsent error message requests = let fatal = ref false in let pending = Queue.create () in List.iter (fun request -> Queue.push request pending) requests; while not (Queue.is_empty pending) do let request = Queue.pop pending in - let retired = Journal_graph_runtime.fail_request runtime request ~message in + let retired = Journal_graph_runtime.fail_request ~error runtime request ~message in responses := Rrbvec.append_list !responses retired.responses; (* Owned durable business failures retain their retry obligation. A refusal of any other work still reports a transport failure. *) - if (retired.responses=[] && retired.requests=[]) || not(List.for_all(fun response-> + let business_responses=List.filter(fun response->match response.Journal_graph_runtime.payload with + | Diagnostic _->false | _->true) retired.responses in + if (business_responses=[] && retired.requests=[]) || not(List.for_all(fun response-> match response.Journal_graph_runtime.payload with | Child_waiting _ | Capture_waiting _ | Mutation_waiting _ | Child_not_admitted _ | Capture_not_admitted _ -> true - | _ -> false) retired.responses) then fatal:=true; + | _ -> false) business_responses) then fatal:=true; (* A failed completion can release queued work. It is still unsent and must be settled, not retried against the refusing Worker. *) List.iter (fun request -> Queue.push request pending) retired.requests @@ -44,9 +48,15 @@ let deliver ~runtime ~send (output : Journal_graph_runtime.output) = | request :: rest -> (match send request with | Accepted -> send_pending rest - | (Full | Not_ready | Stopping) as result -> + | (Full | Not_ready | Stopping | Refused _) as result -> let message = Option.get (error_message result) in - if retire_unsent message (request :: rest) then Some message else None) + let error=match result with + |Refused error->error + |_->let reason=match result with + |Full->Journal_worker_coordination.Worker_api.Full + |Not_ready->Not_ready |Stopping->Stopping |Accepted|Refused _->assert false in + Journal_worker_coordination_host.admission_diagnostic reason in + if retire_unsent error message (request :: rest) then Some message else None) in let error = send_pending output.requests in { responses = Rrbvec.to_list !responses; error } diff --git a/app/journal_graph_transport.mli b/app/journal_graph_transport.mli index 6e349a08..139626e3 100644 --- a/app/journal_graph_transport.mli +++ b/app/journal_graph_transport.mli @@ -3,6 +3,7 @@ type send_result = | Full | Not_ready | Stopping + | Refused of Logseq_db_worker.Error.t type delivery = { responses : Journal_graph_runtime.response list diff --git a/app/journal_lui_bridge.c b/app/journal_lui_bridge.c index 06417cb2..d111e881 100644 --- a/app/journal_lui_bridge.c +++ b/app/journal_lui_bridge.c @@ -49,11 +49,10 @@ static lui_patch_callback patch_callback = NULL; static journal_wakeup_callback wakeup_callback = NULL; static journal_platform_request_callback platform_request_callback = NULL; -static void report_ocaml_exception(const char *where, value result) { - (void)result; +static void report_ocaml_exception(const char *where, int reason) { /* Exception text may contain credentials, paths or graph content. The return code reaches the native host, which can report failure without a patch. */ - fprintf(stderr, "journal_lui_bridge: operation=%s code=ocamlCallbackFailed\n", where); + fprintf(stderr, "journal_lui_bridge: operation=%s code=ocamlCallbackFailed reason=%d\n", where, reason); } /* Only the OCaml owner's exact allowlist is queried. Never format an exception @@ -68,7 +67,10 @@ static int classify_exception(value result) { int reason = 2; if (!Is_exception_result(classified) && Is_long(classified)) { int candidate = Int_val(classified); - if (candidate >= 5 && candidate <= 7) reason = candidate; + if ((candidate >= 5 && candidate <= 7) || + (candidate >= 500 && candidate <= 531) || + (candidate >= 600 && candidate <= 631) || + (candidate >= 700 && candidate <= 731)) reason = candidate; } CAMLreturnT(int,reason); } @@ -92,8 +94,8 @@ static patch_response copy_patch(const char *where, value result) { patch_response response = pending_response(where,0); response.callback = patch_callback; if (Is_exception_result(result)) { - report_ocaml_exception(where, result); response.reason = classify_exception(result); + report_ocaml_exception(where, response.reason); return response; } size_t length = caml_string_length(result); @@ -388,7 +390,7 @@ LUI_EXPORT int64_t lui_ocaml_root_node(void) { if (root != NULL) { value result = caml_callback_exn(*root, Val_unit); if (Is_exception_result(result)) { - report_ocaml_exception("lui_ocaml_root_node", result); + report_ocaml_exception("lui_ocaml_root_node", classify_exception(result)); } else { node = (int64_t)Long_val(result); } @@ -446,8 +448,8 @@ static int32_t deliver_platform(const char *name, const char *data, value payload = copy_bytes(data, length); value result = caml_callback_exn(*handler, payload); if (Is_exception_result(result)) { - report_ocaml_exception(name, result); reason = classify_exception(result); + report_ocaml_exception(name, reason); } else { accepted = 1; reason = 0; } diff --git a/app/journal_manager_view.ml b/app/journal_manager_view.ml index c5b2bbc1..05e8c8f1 100644 --- a/app/journal_manager_view.ml +++ b/app/journal_manager_view.ml @@ -234,7 +234,7 @@ let view ?picker_signal props ~on_action = else ( match presence owner with | Some (Ok Graph_service.Not_downloaded) -> "Download and open" - | Some (Error _) -> "Unable to check local copy" + | Some (Error message) -> message | _ -> "") in let selection_enabled owner = owner.deleting = None in diff --git a/app/journal_worker_coordination_host.ml b/app/journal_worker_coordination_host.ml index 68a66b83..98033d20 100644 --- a/app/journal_worker_coordination_host.ml +++ b/app/journal_worker_coordination_host.ml @@ -2,6 +2,86 @@ module Worker = Journal_worker_coordination.Worker_api module Core = Journal_worker_coordination.Worker_pure_coordinator module Runner = Journal_worker_coordination.Worker_effect_runner +let error_message = function + | Worker.Db_error error -> Logseq_db_worker.Error.message error + | Invalid_input message | Service_failed message | Unsupported_operation message + | Protocol_mismatch message | Stale_read message | Resync_required message -> message + | Revoked_session -> "The graph session changed." + | Host_change_in_progress -> "The graph is closing. The outcome is retained." + | Response_budget_exceeded -> "The result exceeded its response budget." +;; + +let diagnostic_error error = + let module E=Logseq_db_worker.Error in + match error with + | Worker.Db_error error -> error + | _ -> + let code,category=match error with + | Worker.Revoked_session->E.Closed_session,"revokedSession" + | Host_change_in_progress->E.Closed_session,"hostChangeInProgress" + | Response_budget_exceeded->E.Response_too_large,"responseBudgetExceeded" + | Invalid_input _->E.Invalid_request,"invalidInput" + | Protocol_mismatch _->E.Invalid_request,"protocolMismatch" + | Unsupported_operation _->E.Unsupported_semantics,"unsupportedOperation" + | Stale_read _->E.Stale_read_cursor,"staleRead" + | Resync_required _->E.Stale_read_cursor,"resyncRequired" + | Service_failed _->E.Closed_session,"serviceFailed" + | Db_error _->assert false in + let origin=E.create_cause_or_fallback ~component:E.Worker_service + ~operation:"coordination" ~code:(Some category) ~message:(error_message error) + ~fallback_message:"The Worker reported an unsafe or oversized failure description." in + E.create_with_origin ~code ~message:origin.message ~origin ~details:[] |> Result.get_ok +;; + +let admission_error = function + | Worker.Full -> Worker.Service_failed "Worker request queue is full" + | Not_ready -> Worker.Service_failed "Worker is not ready" + | Stopping -> Worker.Service_failed "Worker is stopping" + | Unsupported_request message -> Worker.Unsupported_operation message +;; + +let admission_diagnostic reason = + let module E=Logseq_db_worker.Error in + let error=diagnostic_error(admission_error reason) in + let code,category=match reason with + |Worker.Full->E.Storage_busy,"queueFull" + |Not_ready->E.Invalid_request,"notReady" + |Stopping->E.Closed_session,"stopping" + |Unsupported_request _->E.Unsupported_semantics,"unsupportedRequest" in + let origin=E.create_cause ~component:E.Worker_service ~operation:"requestAdmission" + ~code:(Some category) ~message:(E.message error) |> Result.get_ok in + E.create_with_origin ~code ~message:(E.message error) ~origin ~details:[] |> Result.get_ok +;; + +let interruption_error = function + | Worker.Transport_lost error -> error + | Cancelled -> Worker.Service_failed "Worker request was cancelled." + | Worker_stopped -> Worker.Service_failed "Worker stopped before the request completed." +;; + +let interruption_diagnostic reason = + let module E=Logseq_db_worker.Error in + let error=diagnostic_error(interruption_error reason) in + let category,message=match reason with + | Worker.Transport_lost _->"transportLost","Worker transport was lost; execution outcome may be unknown." + | Cancelled->"cancelled","Worker request was cancelled." + | Worker_stopped->"workerStopped","Worker stopped before the request completed." in + let context=E.create_cause ~component:E.Worker_service ~operation:"requestCompletion" + ~code:(Some category) ~message |> Result.get_ok in + E.wrap ~code:(E.code error) ~message:(E.message error) ~details:(E.details error) + ~context error |> Result.get_ok +;; + +let reply_error = function + | Worker.Answered(Error error)->Some error + | Not_admitted reason->Some(Worker.Db_error(admission_diagnostic reason)) + | Interrupted reason->Some(Worker.Db_error(interruption_diagnostic reason)) + | Answered(Ok _)->None +;; +let reply_error_message ~fallback reply = + Option.fold ~none:fallback ~some:error_message (reply_error reply) +;; + type watch = | Watch : 'a Core.ticket @@ -204,5 +284,5 @@ let recover_import_sources host intents ~failed = |Ok ticket->watch host ticket ~f:(function |Worker.Answered(Ok _)->finish None |Answered(Error error)->finish(Some error) - |Not_admitted _|Interrupted _->finish(Some(Worker.Service_failed "The original attachment source could not be read.")))in + |(Not_admitted _|Interrupted _) as reply->finish(reply_error reply))in Rrbvec.iter issue reads diff --git a/app/journal_worker_coordination_host.mli b/app/journal_worker_coordination_host.mli index 2da38874..f021f2b2 100644 --- a/app/journal_worker_coordination_host.mli +++ b/app/journal_worker_coordination_host.mli @@ -2,6 +2,17 @@ module Worker = Journal_worker_coordination.Worker_api module Core = Journal_worker_coordination.Worker_pure_coordinator module Runner = Journal_worker_coordination.Worker_effect_runner +(** Presentation text plus the complete typed diagnostic for error ledgers. + Database causes and context are retained without reconstructing them. *) +val error_message : Worker.error -> string +val diagnostic_error : Worker.error -> Logseq_db_worker.Error.t +val admission_error : Worker.admission_error -> Worker.error +val admission_diagnostic : Worker.admission_error -> Logseq_db_worker.Error.t +val interruption_error : Worker.interruption -> Worker.error +val interruption_diagnostic : Worker.interruption -> Logseq_db_worker.Error.t +val reply_error : 'a Worker.reply -> Worker.error option +val reply_error_message : fallback:string -> 'a Worker.reply -> string + module Pure_source_recovery : sig type t type read = private {session:Worker.session;intent:Worker.asset_import_intent;attempt:int} diff --git a/app/spec/worker_coordination/worker_api.mli b/app/spec/worker_coordination/worker_api.mli index 7ef42401..87b8cc61 100644 --- a/app/spec/worker_coordination/worker_api.mli +++ b/app/spec/worker_coordination/worker_api.mli @@ -444,7 +444,7 @@ val maximum_mutation_recovery_bytes : int type interruption = Worker_effect_authority.interruption = | Cancelled - | Transport_lost + | Transport_lost of error | Worker_stopped type 'a reply = 'a Worker_effect_authority.reply = diff --git a/apple-tests/authentication/JournalCognitoNativeReasonTests.swift b/apple-tests/authentication/JournalCognitoNativeReasonTests.swift index 2df340e3..1701c88e 100644 --- a/apple-tests/authentication/JournalCognitoNativeReasonTests.swift +++ b/apple-tests/authentication/JournalCognitoNativeReasonTests.swift @@ -96,7 +96,21 @@ private enum FixtureSecurity { FixtureSecurity.state.withLock { $0.data = Data(repeating: 65, count: 131073) } try await rejected(["keychainLoad.responseTooLarge"]) { _ = try await storage.load() } FixtureSecurity.state.withLock { $0.data = Data("private-token-sentinel".utf8) } - try await rejected(["keychainLoad.invalidRecord"]) { _ = try await storage.load() } + try await rejected(["keychainLoad.invalidRecord", "decodeCorrupted"]) { _ = try await storage.load() } + FixtureSecurity.state.withLock { $0.data = Data("{}".utf8) } + try await rejected(["keychainLoad.invalidRecord", "decodeMissingKey"]) { _ = try await storage.load() } + FixtureSecurity.state.withLock { $0.data = Data(#"{"idToken":42}"#.utf8) } + try await rejected(["keychainLoad.invalidRecord", "decodeTypeMismatch"]) { _ = try await storage.load() } + for (payload, expected) in [("{}", "decodeMissingKey"), ("{", "decodeCorrupted"), + (#"{"version":"private-token-sentinel"}"#, "decodeTypeMismatch"), + (#"{"version":2}"#, "decodeMissingKey")] { + do { _ = try JournalLocalAccountBindingStore.decode(Data(payload.utf8)); preconditionFailure("accepted corrupt binding") } + catch { + let issue = JournalAccountIssue.classify(error, stage: .bindingRead) + precondition(issue.rejectsBinding && issue.diagnostic.contains(expected), "binding lost decode cause") + precondition(!issue.diagnostic.contains("private-")) + } + } FixtureSecurity.state.withLock { $0.updateStatus = -25308 } try await rejected(["keychainSave.status", "osstatus -25308"]) { try await storage.save(tokens) } FixtureSecurity.state.withLock { $0.updateStatus = errSecItemNotFound; $0.status = -25299 } @@ -123,6 +137,17 @@ private enum FixtureSecurity { FixtureBrowserState.error = NSError(domain: "private-token-sentinel", code: 77, userInfo: [NSLocalizedDescriptionKey: "private-token-sentinel"]) try await rejected(["browser.browserFailed"]) { _ = try await browser.authenticate(url, scheme: "synthetic") } + FixtureBrowserState.error = ASWebAuthenticationSessionError(.presentationContextInvalid) + try await rejected(["browser.browserFailed", "webAuthentication", "code 3"]) { + _ = try await browser.authenticate(url, scheme: "synthetic") + } + let wrapped = NSError(domain: NSCocoaErrorDomain, code: 256, + userInfo: [NSUnderlyingErrorKey: NSError(domain: NSPOSIXErrorDomain, code: 13), + NSLocalizedDescriptionKey: "private-token-sentinel", NSFilePathErrorKey: "/private-token-sentinel"]) + FixtureBrowserState.error = wrapped + try await rejected(["browser.browserFailed", "cocoa", "code 256", "posix", "code 13"]) { + _ = try await browser.authenticate(url, scheme: "synthetic") + } FixtureBrowserState.error = ASWebAuthenticationSessionError(.canceledLogin) do { _ = try await browser.authenticate(url, scheme: "synthetic"); preconditionFailure("cancel returned success") } catch is CancellationError {} diff --git a/apple-tests/authentication/JournalCognitoTests.swift b/apple-tests/authentication/JournalCognitoTests.swift index f89d4916..849f7934 100644 --- a/apple-tests/authentication/JournalCognitoTests.swift +++ b/apple-tests/authentication/JournalCognitoTests.swift @@ -68,6 +68,10 @@ import Security .queryItems!.map { ($0.name, $0.value ?? "") }) } func token(_ use: String) throws -> String { + if defect == "tokenSize" { return String(repeating: "a", count: 32769) } + if defect == "tokenShape" { return "a.b" } + if defect == "base64" { return "%.b.c" } + var claims: [String: Any] = ["iss": defect == "issuer" ? "https://evil.example" : config.issuer, "sub": "fixture-user", "exp": now.addingTimeInterval(lifetime).timeIntervalSince1970, "iat": now.timeIntervalSince1970, "token_use": use] @@ -75,7 +79,15 @@ import Security claims["aud"] = defect == "audience" ? "other" : config.clientID claims["nonce"] = defect == "nonce" ? "wrong" : query(authorization!)["nonce"] } else { claims["client_id"] = config.clientID } - let input = Self.url64(try JSONSerialization.data(withJSONObject: ["alg": "RS256", "kid": "test-key"])) + if defect == "claims" { claims["iss"] = 42 } + if defect == "use" { claims["token_use"] = "unexpected" } + if defect == "issued" { claims["iat"] = now.addingTimeInterval(61).timeIntervalSince1970 } + if defect == "subject" { claims["sub"] = "" } + if defect == "subjectMismatch" && use == "access" { claims["sub"] = "other" } + var header: [String: Any] = ["alg": defect == "header" ? 42 : "RS256", "kid": "test-key"] + if defect == "algorithm" { header["alg"] = "HS256" } + if defect == "critical" { header["crit"] = NSNull() } + let input = Self.url64(try JSONSerialization.data(withJSONObject: header)) + "." + Self.url64(try JSONSerialization.data(withJSONObject: claims)) var signature = SecKeyCreateSignature(key, .rsaSignatureMessagePKCS1v15SHA256, Data(input.utf8) as CFData, nil)! as Data if defect == "signature" { signature[0] ^= 1 } @@ -111,6 +123,9 @@ import Security var object: [String: Any] = ["id_token": try self.token("id"), "access_token": try self.token("access"), "expires_in": Int(self.lifetime), "token_type": "Bearer"] if !self.omitRefresh { object["refresh_token"] = "synthetic-refresh+&=" } + if self.defect == "tokenType" { object["token_type"] = "unexpected" } + if self.defect == "expires" { object["expires_in"] = 0 } + if self.defect == "refreshSize" { object["refresh_token"] = String(repeating: "a", count: 32769) } data = try JSONSerialization.data(withJSONObject: object) } } @@ -187,11 +202,11 @@ import Security switch mode { case "tokenHTTP": f.status = 503; f.tokenError = "server_error"; expected = ["token.httpStatus", "http 503", "oauth.server_error"] case "unknownOAuth": f.status = 401; f.tokenError = "private-token-sentinel"; expected = ["token.httpStatus", "http 401", "oauth.unknown"] - case "tokenShape": f.tokenData = Data("{\"id_token\":\"private-token-sentinel\"}".utf8); expected = ["token.responseShape"] + case "tokenShape": f.tokenData = Data("{\"id_token\":\"private-token-sentinel\"}".utf8); expected = ["token.responseShape", "decodeMissingKey"] case "tokenSize": f.tokenData = Data(repeating: 65, count: 131073); expected = ["token.responseTooLarge"] case "redirect": f.redirect = true; expected = ["token.redirect"] case "jwksHTTP": f.jwksStatus = 502; expected = ["jwks.httpStatus", "http 502"] - case "jwksShape": f.jwksData = Data("private-token-sentinel".utf8); expected = ["jwks.responseShape"] + case "jwksShape": f.jwksData = Data("private-token-sentinel".utf8); expected = ["jwks.responseShape", "decodeCorrupted"] default: f.jwksData = Data(repeating: 65, count: 262145); expected = ["jwks.responseTooLarge"] } do { try await f.session().signIn(); preconditionFailure("Expected \(mode) rejection") } @@ -207,7 +222,7 @@ import Security for (error, expectedKind, reason) in [ (URLError(.timedOut) as Error, JournalAccountIssue.Kind.network, "url -1001"), (NSError(domain: NSOSStatusErrorDomain, code: -25308) as Error, .secureStorageUnavailable, "osstatus -25308"), - (NSError(domain: "private-token-sentinel", code: 77) as Error, .unavailable, "token.transport")] { + (NSError(domain: "private-token-sentinel", code: 77) as Error, .unavailable, "cause.other · code 77")] { let f = try Issuer(); f.transportError = error do { try await f.session().signIn(); preconditionFailure("transport failure admitted") } catch { @@ -251,12 +266,56 @@ import Security else { try await s.signIn(); precondition(f.stored != nil) } } print("PASS callback mismatch, duplicate/mixed parameters, cancel/error/expiry/replay/sign-out and retry") - for mode in ["nonce", "audience", "issuer", "signature", "expired", "refresh"] { + for mode in ["nonce", "audience", "issuer", "signature", "expired", "refresh", "header", "claims", "tokenSize", "tokenShape", "base64", "algorithm", "critical", "use", "issued", "subject", "subjectMismatch", "tokenType", "expires", "refreshSize", "keyMissing", "keyDuplicate", "keyType", "keyAlgorithm", "keyUse", "keySize"] { let f = try Issuer(); f.defect = mode + if mode.hasPrefix("key") { + let object = try JSONSerialization.jsonObject(with: f.jwks) as! [String: [[String: Any]]] + var key = object["keys"]![0] + if mode == "keyMissing" { key["kid"] = "other" } + if mode == "keyType" { key["kty"] = "EC" } + if mode == "keyAlgorithm" { key["alg"] = "HS256" } + if mode == "keyUse" { key["use"] = "enc" } + if mode == "keySize" { key["n"] = "YQ" } + f.jwksData = try JSONSerialization.data(withJSONObject: ["keys": mode == "keyDuplicate" ? [key,key] : [key]]) + } if mode == "expired" { f.lifetime = -1 }; if mode == "refresh" { f.omitRefresh = true } - let s = f.session(); try await rejects { try await s.signIn() }; precondition(f.stored == nil) + let s = f.session() + do { try await s.signIn(); preconditionFailure("invalid token admitted") } + catch { + let expected = ["nonce": "nonceMismatch", "audience": "audienceMismatch", + "issuer": "issuerMismatch", "signature": "signatureInvalid", + "expired": "tokenExpired", "refresh": "refreshTokenMissing", + "header": "tokenHeader.responseShape", "claims": "tokenClaims.responseShape", + "tokenSize": "tokenTooLarge", "tokenShape": "tokenSegmentsInvalid", "base64": "base64Invalid", + "algorithm": "algorithmUnsupported", "critical": "criticalHeaderUnsupported", "use": "tokenUseMismatch", + "issued": "issuedAtInvalid", "subject": "subjectInvalid", "subjectMismatch": "subjectMismatch", + "tokenType": "tokenTypeInvalid", "expires": "expiresInInvalid", "refreshSize": "refreshTokenTooLarge", + "keyMissing": "signingKeyMissing", "keyDuplicate": "signingKeyAmbiguous", "keyType": "signingKeyTypeInvalid", + "keyAlgorithm": "signingKeyAlgorithmInvalid", "keyUse": "signingKeyUseInvalid", "keySize": "signingKeySizeInvalid"][mode]! + let issue = JournalAccountIssue.classify(error, stage: .identityLookup) + if mode == "header" || mode == "claims" { precondition(issue.diagnostic.contains("decodeTypeMismatch")) } + precondition(issue.diagnostic.contains(expected), "validation lost cause: \(issue.diagnostic)") + precondition(!issue.diagnostic.contains("https://") && !issue.diagnostic.contains("fixture-user")) + } + precondition(f.stored == nil) } print("PASS invalid signature/issuer/audience/nonce/expiry/incomplete tokens") + for (field, reason) in [("issuer", "issuerMismatch"), ("client", "audienceMismatch"), + ("user", "subjectInvalid"), ("id", "idTokenMissing"), ("access", "accessTokenMissing"), ("refresh", "refreshTokenMissing")] { + let f = try Issuer() + f.stored = JournalCognitoTokens(idToken: field == "id" ? "" : "synthetic", + accessToken: field == "access" ? "" : "synthetic", refreshToken: field == "refresh" ? "" : "synthetic", + userID: field == "user" ? "" : "synthetic", issuer: field == "issuer" ? "PRIVATE" : f.config.issuer, + clientID: field == "client" ? "PRIVATE" : f.config.clientID, expiresAt: f.now.addingTimeInterval(300)) + do { _ = try await f.session().currentUserID(); preconditionFailure("invalid restored session admitted") } + catch { + let issue = JournalAccountIssue.classify(error, stage: .identityLookup) + precondition(issue.diagnostic.contains("sessionRestore." + reason), "restore lost reason") + precondition(!issue.diagnostic.contains("PRIVATE")) + precondition(f.requests.isEmpty, "invalid restored session reached network") + } + } + let restored = issuer.session(); let restoredUser = try await restored.currentUserID(); precondition(restoredUser == user) issuer.now = issuer.now.addingTimeInterval(3550); issuer.holdTokens = true; issuer.omitRefresh = true let first = Task { try await restored.freshIDToken() }; while issuer.tokenHold == nil { await Task.yield() } diff --git a/apple-tests/native-events/JournalLUIEventsTests.swift b/apple-tests/native-events/JournalLUIEventsTests.swift index 6bdc0db2..c1b14ef7 100644 --- a/apple-tests/native-events/JournalLUIEventsTests.swift +++ b/apple-tests/native-events/JournalLUIEventsTests.swift @@ -155,7 +155,9 @@ private let receivePatch: PatchCallback = { source in "OCaml exception reported success") try check(patches.isEmpty, "OCaml exception emitted a patch") try check(failureReason(7) == 2 && failureReason(3) == 0, "reason must belong to the exact operation") - for (text, reason) in [("raise-startup",Int32(5)),("raise-worker",6),("raise-mount",7)] { + for (text, reason) in [("raise-startup",Int32(5)),("raise-worker",6),("raise-mount",7), + ("raise-truncated",501),("raise-magic",503),("raise-permission",615),("raise-space",617), + ("raise-private",500),("raise-unix",717)] { try check(JournalLUIEvents.dispatch(.textChanged(node:node,text:text)) == 0, "typed startup exception was accepted") try check(failureReason(7) == reason, "fixed startup exception reason was lost") diff --git a/apple-tests/native-events/bridge_fixture.ml b/apple-tests/native-events/bridge_fixture.ml index 6d23ebf3..5b2b064b 100644 --- a/apple-tests/native-events/bridge_fixture.ml +++ b/apple-tests/native-events/bridge_fixture.ml @@ -64,6 +64,18 @@ let dispatch = function | Press node -> patch node "press" | LongPress node -> patch node "longPress" | TextChanged (_, "raise") -> failwith "native event fixture rejection" + | TextChanged (_, "raise-truncated") -> + raise (Journal_bridge.initialization_failure Decode_startup "startup payload is truncated") + | TextChanged (_, "raise-magic") -> + raise (Journal_bridge.initialization_failure Decode_startup "invalid startup magic") + | TextChanged (_, "raise-permission") -> + raise (Journal_bridge.initialization_failure Start_worker "worker.start.permissionDenied") + | TextChanged (_, "raise-space") -> + raise (Journal_bridge.initialization_failure Start_worker "worker.start.noSpace") + | TextChanged (_, "raise-private") -> + raise (Journal_bridge.initialization_failure Decode_startup "SENSITIVE https://private.invalid/?token=SECRET") + | TextChanged (_, "raise-unix") -> + raise (Journal_bridge.initialization_exception (Unix.Unix_error (Unix.ENOSPC, "SENSITIVE", "SENSITIVE"))) | TextChanged (_, "raise-startup") -> failwith "application.decodeStartup.failed" | TextChanged (_, "raise-worker") -> failwith "application.startWorker.failed" | TextChanged (_, "raise-mount") -> failwith "application.mount.failed" diff --git a/apple-tests/platform-services/JournalAccountThreadingTests.swift b/apple-tests/platform-services/JournalAccountThreadingTests.swift index 31307c87..2eb476c0 100644 --- a/apple-tests/platform-services/JournalAccountThreadingTests.swift +++ b/apple-tests/platform-services/JournalAccountThreadingTests.swift @@ -106,6 +106,21 @@ enum JournalLocalAccountBindingStore { } @MainActor static func main() async throws { + // Recovery bytes are decoded by the native storage owner, before reducer events. + for (payload, expected) in [("{", "decodeCorrupted"), ("{}", "decodeMissingKey"), + (#"{"version":"PRIVATE"}"#, "decodeTypeMismatch")] { + JournalLocalAccountBindingStore.state.withLock { $0.recovery = Data(payload.utf8) } + let owner = JournalNativeServices.makePlatformServices() + do { _ = try await owner.response(for: .localAccount); preconditionFailure("corrupt recovery admitted") } + catch { + let issue = JournalAccountIssue.classify(error, stage: .bindingWrite) + precondition(issue.diagnostic.contains("recovery.invalidRecord") && issue.diagnostic.contains(expected), + "recovery lost cause: \(issue.diagnostic)") + precondition(!issue.diagnostic.contains("PRIVATE")) + } + } + JournalLocalAccountBindingStore.state.withLock { $0.recovery = nil } + // Configuration encoding is owned by the native startup producer before // any OCaml request. These strings do not access the filesystem/network. for (path,origin) in [("SECRET-relative-path","https://example.invalid"), diff --git a/apple-tests/platform-services/JournalPlatformServicesTests.swift b/apple-tests/platform-services/JournalPlatformServicesTests.swift index f2329c3d..3588318a 100644 --- a/apple-tests/platform-services/JournalPlatformServicesTests.swift +++ b/apple-tests/platform-services/JournalPlatformServicesTests.swift @@ -88,11 +88,15 @@ import Foundation userInfo: [NSLocalizedDescriptionKey: "private-token-sentinel"]) do { _ = try await reasonService.response(for: .authenticatedUser); fatalError("expected unknown failure") } catch { precondition((error as NSError).domain == "private-account-sentinel") } - precondition(reasonService.accountIssue == safeFallback, "unknown account error must keep safe fallback") + let unknownIssue = reasonService.accountIssue + precondition(unknownIssue?.kind == safeFallback.kind && unknownIssue?.stage == safeFallback.stage, + "unknown account error must keep recovery policy") + precondition(unknownIssue?.diagnostic.contains("cause.other · code 73") == true) + precondition(unknownIssue?.diagnostic.contains("private-") == false) reasonAuth.userFailure = CancellationError() do { _ = try await reasonService.response(for: .authenticatedUser); fatalError("expected cancellation") } catch is CancellationError {} - precondition(reasonService.accountIssue == safeFallback, "cancellation must not publish a new account issue") + precondition(reasonService.accountIssue == unknownIssue, "cancellation must not publish a new account issue") let existingIssue = JournalAccountIssue(kind: .unsupportedFormat, stage: .bindingRead, formatVersion: 9) precondition(JournalAccountIssue.classify(existingIssue, stage: .identityLookup) == existingIssue) let old = JournalLocalAccount(userID: "offline-user", managedSyncOrigin: origin) diff --git a/apple-tests/platform-services/JournalRuntimeFailureTests.swift b/apple-tests/platform-services/JournalRuntimeFailureTests.swift index aa617aba..f8db7d25 100644 --- a/apple-tests/platform-services/JournalRuntimeFailureTests.swift +++ b/apple-tests/platform-services/JournalRuntimeFailureTests.swift @@ -26,8 +26,10 @@ let journalAppIcons: [String: String] = [:] @MainActor @main struct JournalRuntimeFailureTests { static func main() async throws { + precondition(JournalRuntime.safeDescription(JournalPlatformWire.Failure.invalidPacket, + operation: "lifecycle").contains("invalidPayload"), "lifecycle packet cause lost") let scenario = CommandLine.arguments[1] - let modes = ["rejected":1, "invalid-patch":2, "pump":3, "platform-event":4, "platform-response":5, "platform-failure":6, "stop-retry":7, "stop-failure":7] + let modes = ["rejected":1, "invalid-patch":2, "pump":3, "platform-event":4, "platform-response":5, "platform-failure":6, "stop-retry":7, "stop-failure":7, "startup-truncated":501, "startup-magic":503, "worker-permission":615, "worker-space":617] setMode(Int32(modes[scenario] ?? 0)) let platform = JournalApplicationPlatform() let runtime = try JournalRuntime(platform: platform, startupPayload: Data(), @@ -86,8 +88,11 @@ let journalAppIcons: [String: String] = [:] let message = reported!.localizedDescription precondition(!message.contains("SENSITIVE"), "Native failure must exclude raw renderer details") precondition(message.contains("runtime."), "Native failure must identify its fixed stage/code") - precondition(message.contains(scenario == "invalid-patch" ? "backendUnsupportedProperty" : "callbackFailed"), + let detail = ["startup-truncated": "startupPayloadTruncated", "startup-magic": "startupMagicInvalid", + "worker-permission": "permissionDenied", "worker-space": "noSpace"] + precondition(message.contains(detail[scenario] ?? (scenario == "invalid-patch" ? "backendUnsupportedProperty" : "callbackFailed")), "native runtime must preserve the safe producer reason") + } print("PASS runtime \(scenario)") } diff --git a/logseq_db_storage/lib/logseq_sqlite_codec.ml b/logseq_db_storage/lib/logseq_sqlite_codec.ml index f8f1cc4e..0f5e7a54 100644 --- a/logseq_db_storage/lib/logseq_sqlite_codec.ml +++ b/logseq_db_storage/lib/logseq_sqlite_codec.ml @@ -9,6 +9,18 @@ type error = | Out_of_range_number of string | Malformed_storage_payload of string +let diagnostic_error error = + let reason, source = match error with + | Unsupported_tag _ -> "codecUnsupportedTag", "" + | Malformed_transit message -> "codecMalformedTransit", message + | Out_of_range_number _ -> "codecNumberRange", "" + | Malformed_storage_payload message -> "codecMalformedPayload", message + in + Logseq_db_types.Failure_diagnostic.of_source + ~operation:"workerOperation" ~reason source + |> Logseq_db_types.Failure_diagnostic.encode +;; + type physical_entry = { address : Datascript.storage_address ; content : string @@ -929,7 +941,7 @@ let encode_physical_batch ?(restore = fun _ -> None) ?metadata entries = let content, addresses = match encode_physical_payload payload with | Ok encoded -> encoded - | Error _ -> invalid_arg ("unable to encode storage address " ^ address) + | Error error -> invalid_arg (diagnostic_error error) in { address; content; addresses }) entries) diff --git a/logseq_db_storage/lib/logseq_sqlite_codec.mli b/logseq_db_storage/lib/logseq_sqlite_codec.mli index 937f9631..32a4a5de 100644 --- a/logseq_db_storage/lib/logseq_sqlite_codec.mli +++ b/logseq_db_storage/lib/logseq_sqlite_codec.mli @@ -43,3 +43,6 @@ val encode_physical_batch -> ?metadata:root_index_metadata -> (Datascript.storage_address * Datascript.storage_payload) list -> (physical_entry list, error) result + +(** Preserve the typed codec category without retaining input values or tags. *) +val diagnostic_error : error -> string diff --git a/logseq_db_storage/lib/logseq_sqlite_storage.ml b/logseq_db_storage/lib/logseq_sqlite_storage.ml index 028a8c3b..59753384 100644 --- a/logseq_db_storage/lib/logseq_sqlite_storage.ml +++ b/logseq_db_storage/lib/logseq_sqlite_storage.ml @@ -109,14 +109,61 @@ let verify_writable_pragmas db = "NORMAL" ;; +(* Keep address/path/value text inside storage; expose only the typed stage and + the bounded source classification across the existing string boundary. *) +let diagnostic_error ~operation error = + let reason,source=match error with + | Pragma_mismatch s -> "pragmaMismatch",s + | Corrupt_storage s -> "storageCorrupt",s + | Begin_failed s -> "storageBeginFailed",s + | Write_failed{message;_} -> "storageWriteFailed",message + | Commit_failed s -> "storageCommitFailed",s + | Checkpoint_failed s -> "storageCheckpointFailed",s + | Close_failed s -> "storageCloseFailed",s in + Logseq_db_types.Failure_diagnostic.of_source ~operation ~reason source + |> Logseq_db_types.Failure_diagnostic.encode +;; + +let commit_stage_error reason source = + Commit_failed + (Logseq_db_types.Failure_diagnostic.of_source ~operation:"workerOperation" ~reason source + |> Logseq_db_types.Failure_diagnostic.encode) +;; + +let exception_source exn = + match exn with + | Invalid_argument source | Failure source + when Option.is_some (Logseq_db_types.Failure_diagnostic.decode source) -> source + | _ -> Printexc.to_string exn +;; + +let map_error_source f = function + | Pragma_mismatch source -> Pragma_mismatch (f source) + | Corrupt_storage source -> Corrupt_storage (f source) + | Begin_failed source -> Begin_failed (f source) + | Write_failed { address; message } -> Write_failed { address; message = f message } + | Commit_failed source -> Commit_failed (f source) + | Checkpoint_failed source -> Checkpoint_failed (f source) + | Close_failed source -> Close_failed (f source) +;; + +let rollback_preserving callbacks error = + try callbacks.rollback (); Error error with + | exn -> + let module D = Logseq_db_types.Failure_diagnostic in + let primary = D.of_source ~operation:"workerOperation" ~reason:"persistenceFailed" + (diagnostic_error ~operation:"workerOperation" error) in + let secondary = D.of_source ~operation:"workerOperation" ~reason:"storageRollbackFailed" + (exception_source exn) in + let source = D.with_related primary secondary |> D.encode in + Error (map_error_source (fun _ -> source) error) +;; + let commit_batch ?outbox_delta callbacks batch = match callbacks.begin_immediate () with | Error message -> Error (Begin_failed message) | Ok () -> - let rollback error = - callbacks.rollback (); - Error error - in + let rollback = rollback_preserving callbacks in let rec write_all = function | [] -> Ok () | write :: rest -> @@ -133,7 +180,7 @@ let commit_batch ?outbox_delta callbacks batch = | Some metadata -> callbacks.upsert_sync_metadata metadata in (match metadata with - | Error message -> rollback (Commit_failed message) + | Error message -> rollback (commit_stage_error "storageMetadataFailed" message) | Ok () -> let outbox = match outbox_delta, batch.sync_outbox with @@ -142,7 +189,7 @@ let commit_batch ?outbox_delta callbacks batch = | None, Some records -> callbacks.replace_sync_outbox records in (match outbox with - | Error message -> rollback (Commit_failed message) + | Error message -> rollback (commit_stage_error "storageOutboxFailed" message) | Ok () -> let receipts = match batch.mutation_receipts with @@ -150,7 +197,7 @@ let commit_batch ?outbox_delta callbacks batch = | Some rows -> callbacks.upsert_mutation_receipts rows in (match receipts with - | Error message -> rollback (Commit_failed message) + | Error message -> rollback (commit_stage_error "storageMutationReceiptFailed" message) | Ok () -> let terminal_receipts = match batch.terminal_batch_receipts with @@ -158,7 +205,7 @@ let commit_batch ?outbox_delta callbacks batch = | Some rows -> callbacks.upsert_terminal_batch_receipts rows in (match terminal_receipts with - | Error message -> rollback (Commit_failed message) + | Error message -> rollback (commit_stage_error "storageTerminalReceiptFailed" message) | Ok () -> (match callbacks.commit () with | Ok () -> Ok () @@ -177,6 +224,7 @@ let commit_sync_metadata callbacks metadata = ;; let unreachable_addresses (callbacks : callbacks) = + try let storage = callbacks.storage in let reachable = Hashtbl.create 256 in let rec visit address = @@ -209,6 +257,7 @@ let unreachable_addresses (callbacks : callbacks) = List.filter (fun address -> not (Hashtbl.mem reachable address)) addresses in Ok (Hashtbl.length reachable, addresses, unreachable) + with exn -> Error (Corrupt_storage (exception_source exn)) ;; let garbage_stats callbacks = @@ -233,10 +282,7 @@ let collect_garbage (callbacks : callbacks) = match callbacks.begin_immediate () with | Error message -> Error (Begin_failed message) | Ok () -> - let rollback error = - callbacks.rollback (); - Error error - in + let rollback = rollback_preserving callbacks in (match try callbacks.storage.storage_delete unreachable; @@ -253,15 +299,19 @@ let collect_garbage (callbacks : callbacks) = ;; let checkpoint callbacks = - match callbacks.checkpoint () with - | Ok () -> Ok () - | Error message -> Error (Checkpoint_failed message) + try + match callbacks.checkpoint () with + | Ok () -> Ok () + | Error message -> Error (Checkpoint_failed message) + with exn -> Error (Checkpoint_failed (exception_source exn)) ;; let close callbacks = - match callbacks.close () with - | Ok () -> Ok () - | Error message -> Error (Close_failed message) + try + match callbacks.close () with + | Ok () -> Ok () + | Error message -> Error (Close_failed message) + with exn -> Error (Close_failed (exception_source exn)) ;; let rc_result db rc = @@ -359,7 +409,7 @@ let open_database path = Logseq_sqlite_codec.decode_physical_payload ~content ~addresses with | Ok payload -> Some payload - | Error _ -> invalid_arg ("invalid storage payload at " ^ address)) + | Error error -> invalid_arg (Logseq_sqlite_codec.diagnostic_error error)) | DONE -> None | rc -> Sqlite3.Rc.check rc; @@ -473,14 +523,7 @@ let open_database path = (match encoded with | Error error -> Error - (match error with - | Unsupported_tag tag -> - "unable to encode staged physical storage batch: unsupported tag " - ^ tag - | Malformed_transit message - | Out_of_range_number message - | Malformed_storage_payload message -> - "unable to encode staged physical storage batch: " ^ message) + (Logseq_sqlite_codec.diagnostic_error error) | Ok encoded -> Ok { writes = @@ -556,7 +599,7 @@ let open_database path = ; upsert_terminal_batch_receipts = Mutation_receipt_store.upsert_terminal_batches sqlite ; commit = exec "COMMIT" - ; rollback = (fun () -> ignore (Sqlite3.exec sqlite "ROLLBACK")) + ; rollback = (fun () -> Sqlite3.Rc.check (Sqlite3.exec sqlite "ROLLBACK")) ; unreachable_address_count = (fun () -> match unreachable_address_count_sql with @@ -592,7 +635,7 @@ let open_database path = in Ok { sqlite; storage; callbacks }) with - | exn -> Error (Begin_failed (Printexc.to_string exn)) + | exn -> Error (Begin_failed (exception_source exn)) ;; let datascript_storage connection = connection.storage @@ -601,6 +644,7 @@ let verify_connection_pragmas connection = verify_writable_pragmas connection.sq let startup_metadata connection = let corrupt message = Error (Corrupt_storage message) in + try match ( connection.storage.storage_restore Datascript.Storage.root_address , connection.storage.storage_restore Datascript.Storage.tail_address ) @@ -621,6 +665,7 @@ let startup_metadata connection = | Some (Storage_root _), None -> corrupt "storage tail address 1 is missing" | Some _, _ -> corrupt "storage root address 0 has the wrong payload" | None, _ -> corrupt "storage root address 0 is missing" + with exn -> corrupt (exception_source exn) ;; let sync_metadata connection = Sync_checkpoint_store.read_database connection.sqlite @@ -662,10 +707,10 @@ let validate_storage_header connection = | Some (Datascript.Storage_tail _) -> Ok () | Some _ -> corrupt "storage tail address 1 has the wrong payload" | None -> corrupt "storage tail address 1 is missing") - | Ok (Datascript.Storage_root _), Error _ -> - corrupt "storage root index metadata is malformed" + | Ok (Datascript.Storage_root _), Error error -> + corrupt (Logseq_sqlite_codec.diagnostic_error error) | Ok _, _ -> corrupt "storage root address 0 has the wrong payload" - | Error _, _ -> corrupt "storage root metadata is malformed") + | Error error, _ -> corrupt (Logseq_sqlite_codec.diagnostic_error error)) with | Invalid_argument message -> corrupt message | exn -> corrupt (Printexc.to_string exn) @@ -684,18 +729,3 @@ let restore_database connection = | Invalid_argument message -> corrupt message | exn -> corrupt (Printexc.to_string exn)) ;; - -(* Keep address/path/value text inside storage; expose only the typed stage and - the bounded source classification across the existing string boundary. *) -let diagnostic_error ~operation error = - let reason,source=match error with - | Pragma_mismatch s -> "pragmaMismatch",s - | Corrupt_storage s -> "storageCorrupt",s - | Begin_failed s -> "storageBeginFailed",s - | Write_failed{message;_} -> "storageWriteFailed",message - | Commit_failed s -> "storageCommitFailed",s - | Checkpoint_failed s -> "storageCheckpointFailed",s - | Close_failed s -> "storageCloseFailed",s in - Logseq_db_types.Failure_diagnostic.of_source ~operation ~reason source - |> Logseq_db_types.Failure_diagnostic.encode -;; diff --git a/logseq_db_storage/lib/storage_session.ml b/logseq_db_storage/lib/storage_session.ml index 9acf5a53..93aa4ced 100644 --- a/logseq_db_storage/lib/storage_session.ml +++ b/logseq_db_storage/lib/storage_session.ml @@ -407,14 +407,8 @@ let stage_transact ?tx_meta t ~authoritative_before tx_ops = stage_transact_batch ?tx_meta t ~authoritative_before [ tx_ops ] ;; -let storage_error_message = function - | Logseq_sqlite_storage.Pragma_mismatch message - | Corrupt_storage message - | Begin_failed message - | Commit_failed message - | Checkpoint_failed message - | Close_failed message -> message - | Write_failed { address; message } -> address ^ ": " ^ message +let storage_error_message error = + Logseq_sqlite_storage.diagnostic_error ~operation:"workerOperation" error ;; let terminalize t message = @@ -588,7 +582,12 @@ let close t = | Error checkpoint, Error close -> Error (Persistence_failed - (storage_error_message checkpoint ^ "; " ^ storage_error_message close))) + (let module D = Logseq_db_types.Failure_diagnostic in + let diagnostic error = + D.of_source ~operation:"closeDatabase" ~reason:"closeFailed" + (storage_error_message error) + in + D.with_related (diagnostic checkpoint) (diagnostic close) |> D.encode))) ;; let is_fatal t = diff --git a/logseq_db_storage/test/test_asset_upload_store.ml b/logseq_db_storage/test/test_asset_upload_store.ml index debad169..9f33de1b 100644 --- a/logseq_db_storage/test/test_asset_upload_store.ml +++ b/logseq_db_storage/test/test_asset_upload_store.ml @@ -187,7 +187,9 @@ let generated_checkpoint_validation () = (replace "[\"Prepared\"]" "\"prepared\"" !payload); reject "invalid UUID rejected" - "Upload checkpoint shape is invalid" + (Logseq_db_types.Failure_diagnostic.create ~operation:"restoreUploads" + ~reason:"invalidContent" ~source:"invalid UUID" () + |> Logseq_db_types.Failure_diagnostic.encode) (replace (U.to_string original.graph) "not-a-uuid" !payload); reject "negative asset size rejected" "Upload checkpoint intent is invalid" (replace "\"size\":4" "\"size\":-1" !payload); reject @@ -528,13 +530,34 @@ let legacy_capture_never_captures_new_input_rows () = (Option.is_some(get(R.read db ~owner ~operation:(uuid 887)))))) ;; +(* The pure checkpoint codec owns generated JSON field validation, not a reducer. *) +let checkpoint_field_error_survives () = + let module D = Logseq_db_types.Failure_diagnostic in + let payload = I.Pure_codec.encode (intent 991) in + let before = "\"title\":\"Fixture image\"" in + let rec replace offset = + if offset + String.length before > String.length payload then Alcotest.fail "title fixture missing" + else if String.sub payload offset (String.length before) = before then + String.sub payload 0 offset ^ "\"title\":[\"SECRET-invalid-title\"]" + ^ String.sub payload (offset + String.length before) + (String.length payload - offset - String.length before) + else replace (offset + 1) in + match I.Pure_codec.decode (replace 0) with + | Ok _ -> Alcotest.fail "invalid title was accepted" + | Error source -> + let diagnostic = D.of_source ~operation:"restoreUploads" ~reason:"invalidContent" source in + Alcotest.(check string) "concrete failing field" "upload-checkpoint-field-title" (D.source_class diagnostic); + Alcotest.(check bool) "raw field value excluded" false (String.contains (D.encode diagnostic) 'S') +;; + let () = Alcotest.run "asset upload storage" [ ( "SQLite" , List.map (fun (n, f) -> Alcotest.test_case n `Quick f) - [ "account cleanup rollback", account_cleanup_rolls_back_on_checkpoint_failure + [ "checkpoint field cause", checkpoint_field_error_survives + ; "account cleanup rollback", account_cleanup_rolls_back_on_checkpoint_failure ; "account cleanup retires complete inputs", account_cleanup_retires_complete_inputs ; "foreign persisted input evidence rejected", input_persisted_foreign_evidence_is_not_adoption ; "legacy manifest preserves new source records", legacy_capture_never_captures_new_input_rows diff --git a/logseq_db_types/lib/asset_upload_intent.ml b/logseq_db_types/lib/asset_upload_intent.ml index 0b1d4658..8d58a69f 100644 --- a/logseq_db_types/lib/asset_upload_intent.ml +++ b/logseq_db_types/lib/asset_upload_intent.ml @@ -173,7 +173,12 @@ let decode payload = Json_codec.decode checkpoint_of_yojson (Yojson.Safe.from_string payload) - |> Result.map_error (fun _ -> "Upload checkpoint shape is invalid") + |> Result.map_error (fun source -> + let diagnostic = Failure_diagnostic.of_source + ~operation:"restoreUploads" ~reason:"invalidContent" source in + if Failure_diagnostic.source_class diagnostic = "unknown" + then "Upload checkpoint shape is invalid" + else Failure_diagnostic.encode diagnostic) in if value.format_version <> 1 then Error "Unsupported upload checkpoint format" diff --git a/logseq_db_types/lib/failure_diagnostic.ml b/logseq_db_types/lib/failure_diagnostic.ml index 7bf39408..e345a23b 100644 --- a/logseq_db_types/lib/failure_diagnostic.ml +++ b/logseq_db_types/lib/failure_diagnostic.ml @@ -4,6 +4,7 @@ type t = ; source_name : string ; cursor : string option ; http_status : int option + ; related : t option } let operations = @@ -28,6 +29,9 @@ let operations = ; "prepareCatalog", "storage" ; "platformCrypto", "crypto" ; "decryptValue", "crypto" + ; "encryptValue", "crypto" + ; "httpTransport", "network" + ; "websocketTransport", "network" ; "decodeSnapshot", "sync" ; "assetCache", "asset" ; "projectGraph", "presentation" @@ -101,13 +105,104 @@ let reasons = ; "storageWriteFailed", "SQLite storage write failed" ; "storageCommitFailed", "SQLite transaction could not commit" ; "storageCheckpointFailed", "SQLite checkpoint failed" + ; "storageRollbackFailed", "SQLite rollback failed" ; "storageCloseFailed", "SQLite storage could not close" + ; "storageMetadataFailed", "SQLite sync metadata update failed" + ; "storageOutboxFailed", "SQLite outbox update failed" + ; "storageMutationReceiptFailed", "SQLite mutation receipt update failed" + ; "storageTerminalReceiptFailed", "SQLite terminal receipt update failed" + ; "codecUnsupportedTag", "Storage codec encountered an unsupported Transit tag" + ; "codecMalformedTransit", "Storage codec could not parse Transit" + ; "codecNumberRange", "Storage codec number is out of range" + ; "codecMalformedPayload", "Storage codec payload has an invalid structure" ] ;; (* These are exact producer-owned literals, never substrings of graph values. *) let producer_sources = - [ "empty-submit-group", "empty submit group" + [ "catalog-cache-field-cache-user-id", "catalog cache: field cache_user_id is missing or has an invalid type" + ; "catalog-cache-field-cache-graphs", "catalog cache: field cache_graphs is missing or has an invalid type" + ; "catalog-cache-field-cache-selected-graph", "catalog cache: field cache_selected_graph is missing or has an invalid type" + ; "catalog-cache-field-cache-resident-graphs", "catalog cache: field cache_resident_graphs is missing or has an invalid type" + ; "catalog-cache-field-graph-id", "catalog cache: field graph_id is missing or has an invalid type" + ; "catalog-cache-field-name", "catalog cache: field name is missing or has an invalid type" + ; "catalog-cache-field-schema", "catalog cache: field schema is missing or has an invalid type" + ; "catalog-cache-field-encrypted", "catalog cache: field encrypted is missing or has an invalid type" + ; "catalog-cache-field-major", "catalog cache: field major is missing or has an invalid type" + ; "catalog-cache-field-minor", "catalog cache: field minor is missing or has an invalid type" + ; "catalog-cache-field-exact", "catalog cache: field exact is missing or has an invalid type" + ; "catalog-cache-duplicate-json-field", "catalog cache: duplicate JSON field" + ; "catalog-cache-invalid-utf-8-field-name", "catalog cache: invalid UTF-8 field name" + ; "catalog-cache-invalid-utf-8-string", "catalog cache: invalid UTF-8 string" + ; "catalog-cache-non-finite-json-number", "catalog cache: non-finite JSON number" + ; "catalog-cache-nonstandard-json-value", "catalog cache: nonstandard JSON value" + ; "catalog-cache-json-exceeds-byte-budget", "catalog cache: JSON exceeds byte budget" + ; "catalog-cache-invalid-json-value", "catalog cache: invalid JSON value" + ; "catalog-cache-invalid-shape", "catalog cache: invalid shape" + ; "upload-checkpoint-field-format-version", "Asset_upload_intent.Pure_codec.checkpoint.format_version" + ; "upload-checkpoint-field-operation-id", "Asset_upload_intent.Pure_codec.checkpoint.operation_id" + ; "upload-checkpoint-field-origin", "Asset_upload_intent.Pure_codec.checkpoint.origin" + ; "upload-checkpoint-field-account", "Asset_upload_intent.Pure_codec.checkpoint.account" + ; "upload-checkpoint-field-graph", "Asset_upload_intent.Pure_codec.checkpoint.graph" + ; "upload-checkpoint-field-asset", "Asset_upload_intent.Pure_codec.checkpoint.asset" + ; "upload-checkpoint-field-version", "Asset_upload_intent.Pure_codec.checkpoint.version" + ; "upload-checkpoint-field-title", "Asset_upload_intent.Pure_codec.checkpoint.title" + ; "upload-checkpoint-field-size", "Asset_upload_intent.Pure_codec.checkpoint.size" + ; "upload-checkpoint-field-staged-file", "Asset_upload_intent.Pure_codec.checkpoint.staged_file" + ; "upload-checkpoint-field-replace-reference", "Asset_upload_intent.Pure_codec.checkpoint.replace_reference" + ; "upload-checkpoint-field-target", "Asset_upload_intent.Pure_codec.checkpoint.target" + ; "upload-checkpoint-field-local-mutation", "Asset_upload_intent.Pure_codec.checkpoint.local_mutation" + ; "upload-checkpoint-field-metadata-mutation", "Asset_upload_intent.Pure_codec.checkpoint.metadata_mutation" + ; "upload-checkpoint-field-phase", "Asset_upload_intent.Pure_codec.checkpoint.phase" + ; "upload-checkpoint-field-revision", "Asset_upload_intent.Pure_codec.checkpoint.revision" + ; "storage-payload-map-shape", "unknown storage payload map" + ; "storage-payload-shape", "unknown storage payload" + ; "storage-unresolved-reference", "storage payload cannot contain unresolved refs" + ; "storage-datom-attribute", "datom attr must be a Transit keyword" + ; "storage-datom-shape", "storage datom must be [e a v tx]" + ; "storage-datoms-shape", "storage datoms must be a Transit array" + ; "storage-children-shape", "storage node :children must be a Transit array" + ; "storage-tail-shape", "storage tail must be a Transit array" + ; "storage-transit-map-pairs", "Transit map expects an even number of entries" + ; "storage-transit-cmap-pairs", "Transit cmap expects an even number of entries" + ; "storage-transit-json-standard", "Transit JSON must be standard JSON" + ; "tls-alert-close-notify", "TLS alert: close notify" + ; "tls-alert-unexpected-message", "TLS alert: unexpected message" + ; "tls-alert-bad-record-mac", "TLS alert: bad record mac" + ; "tls-alert-record-overflow", "TLS alert: record overflow" + ; "tls-alert-handshake-failure", "TLS alert: handshake failure" + ; "tls-alert-bad-certificate", "TLS alert: bad certificate" + ; "tls-alert-certificate-expired", "TLS alert: certificate expired" + ; "tls-alert-certificate-unknown", "TLS alert: certificate unknown" + ; "tls-alert-decode-error", "TLS alert: decode error" + ; "tls-alert-protocol-version", "TLS alert: protocol version" + ; "tls-alert-inappropriate-fallback", "TLS alert: inappropriate fallback" + ; "tls-alert-user-canceled", "TLS alert: user canceled" + ; "tls-alert-no-renegotiation", "TLS alert: no renegotiation" + ; "tls-alert-missing-extension", "TLS alert: missing extension" + ; "tls-alert-unsupported-extension", "TLS alert: unsupported extension" + ; "tls-alert-unrecognized-name", "TLS alert: unrecognized name" + ; "tls-alert-no-application-protocol", "TLS alert: no application protocol" + ; "tls-alert-unknown", "TLS alert: unknown" + ; "tls-failure-close-notify", "TLS failure: close notify" + ; "tls-failure-unexpected-message", "TLS failure: unexpected message" + ; "tls-failure-bad-record-mac", "TLS failure: bad record mac" + ; "tls-failure-record-overflow", "TLS failure: record overflow" + ; "tls-failure-handshake-failure", "TLS failure: handshake failure" + ; "tls-failure-bad-certificate", "TLS failure: bad certificate" + ; "tls-failure-certificate-expired", "TLS failure: certificate expired" + ; "tls-failure-certificate-unknown", "TLS failure: certificate unknown" + ; "tls-failure-decode-error", "TLS failure: decode error" + ; "tls-failure-protocol-version", "TLS failure: protocol version" + ; "tls-failure-inappropriate-fallback", "TLS failure: inappropriate fallback" + ; "tls-failure-user-canceled", "TLS failure: user canceled" + ; "tls-failure-no-renegotiation", "TLS failure: no renegotiation" + ; "tls-failure-missing-extension", "TLS failure: missing extension" + ; "tls-failure-unsupported-extension", "TLS failure: unsupported extension" + ; "tls-failure-unrecognized-name", "TLS failure: unrecognized name" + ; "tls-failure-no-application-protocol", "TLS failure: no application protocol" + ; "tls-failure-unknown", "TLS failure: unknown" + ; "empty-submit-group", "empty submit group" ; "duplicate-submit-member", "duplicate submit member" ; "submit-member-missing", "submit member is missing" ; "submit-member-not-queued", "submit member is not queued" @@ -426,6 +521,19 @@ let source_classes = ; "sqlite-aborted", "SQLite operation was aborted" ; "sqlite-too-large", "SQLite value exceeds its size limit" ; "sqlite-type-mismatch", "SQLite value type does not match" + ; "io-error", "Operating system I/O failed" + ; "io-not-directory", "Path component is not a directory" + ; "io-is-directory", "Expected a file but found a directory" + ; "io-readonly", "Filesystem is read-only" + ; "io-exists", "Filesystem object already exists" + ; "io-process-files", "Process file descriptor limit reached" + ; "io-system-files", "System file descriptor limit reached" + ; "network-refused", "Connection was refused" + ; "network-reset", "Connection was reset" + ; "network-timeout", "Connection timed out" + ; "network-unreachable", "Network is unreachable" + ; "network-host-unreachable", "Host is unreachable" + ; "io-broken-pipe", "Pipe or connection is closed" ; "io-no-space", "Storage has no free space" ; "io-permission", "Storage permission was denied" ; "io-missing", "Storage object was not found" @@ -491,6 +599,19 @@ let source_prefixes = ; "CHECK constraint failed:", "sqlite-constraint" ; "no such table:", "sqlite-schema" ; "no such column:", "sqlite-schema" + ; "Unix.Unix_error(Unix.EIO,", "io-error" + ; "Unix.Unix_error(Unix.ENOTDIR,", "io-not-directory" + ; "Unix.Unix_error(Unix.EISDIR,", "io-is-directory" + ; "Unix.Unix_error(Unix.EROFS,", "io-readonly" + ; "Unix.Unix_error(Unix.EEXIST,", "io-exists" + ; "Unix.Unix_error(Unix.EMFILE,", "io-process-files" + ; "Unix.Unix_error(Unix.ENFILE,", "io-system-files" + ; "Unix.Unix_error(Unix.ECONNREFUSED,", "network-refused" + ; "Unix.Unix_error(Unix.ECONNRESET,", "network-reset" + ; "Unix.Unix_error(Unix.ETIMEDOUT,", "network-timeout" + ; "Unix.Unix_error(Unix.ENETUNREACH,", "network-unreachable" + ; "Unix.Unix_error(Unix.EHOSTUNREACH,", "network-host-unreachable" + ; "Unix.Unix_error(Unix.EPIPE,", "io-broken-pipe" ; "Unix.Unix_error(Unix.ENOSPC,", "io-no-space" ; "Unix.Unix_error(Unix.EACCES,", "io-permission" ; "Unix.Unix_error(Unix.EPERM,", "io-permission" @@ -560,6 +681,7 @@ let create ~operation ~reason ?source ?cursor ?http_status () = ; cursor = Option.bind cursor (fun value -> if value < 0L then None else Some (Int64.to_string value)) + ; related = None ; http_status = (match http_status with | Some status when valid_http_status status -> Some status @@ -574,7 +696,13 @@ let reason value = value.reason_name let source_class value = value.source_name let code value = value.operation_name ^ "." ^ value.reason_name -let message value = +let with_related primary secondary = + { primary with related = Some { secondary with related = None } } +;; + +let related value = value.related + +let rec message value = let summary = value.operation_name ^ ": " ^ List.assoc value.reason_name reasons in let cause = if value.source_name = "unknown" @@ -582,13 +710,14 @@ let message value = else ". Cause: " ^ List.assoc value.source_name source_classes in let http = Option.fold ~none:"" ~some:(fun status -> Printf.sprintf " (http=%d)" status) value.http_status in - let text = summary ^ cause ^ http ^ "." in + let additional = Option.fold ~none:"" ~some:(fun other -> " Also: " ^ message other) value.related in + let text = summary ^ cause ^ http ^ "." ^ additional in if String.length text <= 512 then text else summary ^ "." ;; let prefix = "journal-failure:v1:" -let encode value = +let rec encode value = let fields = [ "operation", `String value.operation_name ; "reason", `String value.reason_name @@ -597,11 +726,13 @@ let encode value = @ Option.fold ~none:[] ~some:(fun cursor -> [ "cursor", `String cursor ]) value.cursor @ Option.fold ~none:[] ~some:(fun status -> [ "httpStatus", `Int status ]) value.http_status in + let fields = fields @ Option.fold ~none:[] + ~some:(fun other -> [ "related", `String (encode other) ]) value.related in prefix ^ Yojson.Safe.to_string (`Assoc fields) ;; -let decode source = - if String.length source > 1024 || not (String.starts_with ~prefix source) +let rec decode_at_depth depth source = + if depth > 1 || String.length source > 1024 || not (String.starts_with ~prefix source) then None else ( try @@ -614,12 +745,13 @@ let decode source = match Yojson.Safe.from_string encoded with | `Assoc fields -> let keys = List.map fst fields in - let valid_keys = List.sort String.compare keys in - if + let valid_keys = List.sort String.compare (List.filter ((<>) "related") keys) in + let related_count = List.length (List.filter ((=) "related") keys) in + if related_count > 1 || (depth = 1 && related_count <> 0) || ( valid_keys <> [ "operation"; "reason"; "source" ] && valid_keys <> [ "cursor"; "operation"; "reason"; "source" ] && valid_keys <> [ "httpStatus"; "operation"; "reason"; "source" ] - && valid_keys <> [ "cursor"; "httpStatus"; "operation"; "reason"; "source" ] + && valid_keys <> [ "cursor"; "httpStatus"; "operation"; "reason"; "source" ]) then None else ( match @@ -648,9 +780,13 @@ let decode source = | None -> Some None | Some (`Int value) when valid_http_status value -> Some (Some value) | _ -> None in - (match cursor, http_status with - | Some cursor, Some http_status -> - Some { operation_name; reason_name; source_name; cursor; http_status } + let related = match List.assoc_opt "related" fields with + | None -> Some None + | Some (`String source) -> Option.map Option.some (decode_at_depth (depth + 1) source) + | _ -> None in + (match cursor, http_status, related with + | Some cursor, Some http_status, Some related -> + Some { operation_name; reason_name; source_name; cursor; http_status; related } | _ -> None) | _ -> None) | _ -> None @@ -658,6 +794,8 @@ let decode source = | Yojson.Json_error _ -> None) ;; +let decode = decode_at_depth 0 + let detail_name = "failureDiagnostic" let signal_prefix = "worker.failure:" diff --git a/logseq_db_types/lib/failure_diagnostic.mli b/logseq_db_types/lib/failure_diagnostic.mli index 7f110f10..e68198f1 100644 --- a/logseq_db_types/lib/failure_diagnostic.mli +++ b/logseq_db_types/lib/failure_diagnostic.mli @@ -37,3 +37,8 @@ val encode : t -> string val decode : string -> t option val detail_name : string val signal_prefix : string + +(** Retain one additional independently failed finalization stage. Nested + secondary causes are dropped to keep the carrier bounded. *) +val with_related : t -> t -> t +val related : t -> t option diff --git a/logseq_db_types/lib/json_codec.ml b/logseq_db_types/lib/json_codec.ml index b17f2713..d66412b2 100644 --- a/logseq_db_types/lib/json_codec.ml +++ b/logseq_db_types/lib/json_codec.ml @@ -39,3 +39,17 @@ let decode ?maximum_bytes parser json = | Failure _ | Invalid_argument _ | Yojson.Json_error _ | Stack_overflow -> Error "invalid JSON value" ;; + +(* Only exact producer-owned decoder identities are projected. Parser messages + may contain input values, so an unknown message is never copied to callers. *) +let error_context ~fields message = + match List.assoc_opt message fields with + | Some context -> context + | None -> + if List.mem message + [ "invalid UTF-8 field name"; "duplicate JSON field"; "invalid UTF-8 string" + ; "non-finite JSON number"; "nonstandard JSON value"; "JSON exceeds byte budget" + ; "invalid JSON value" ] + then message + else "invalid shape" +;; diff --git a/logseq_db_worker/lib/effect_runner/effect_runner.ml b/logseq_db_worker/lib/effect_runner/effect_runner.ml index 96e86a50..92defbe0 100644 --- a/logseq_db_worker/lib/effect_runner/effect_runner.ml +++ b/logseq_db_worker/lib/effect_runner/effect_runner.ml @@ -656,6 +656,7 @@ let decrypt_protected_value t key source = ;; let asset_failure_message = function + | Sync.Asset_with_cause (_, detail) -> Logseq_db_types.Failure_diagnostic.message detail | Sync.Asset_invalid_content source -> let module D = Logseq_db_types.Failure_diagnostic in Option.value (D.decode source) @@ -1008,9 +1009,8 @@ let effect_error message = match D.decode message with | None -> worker_error ~code:E.Storage_busy message | Some detail -> - E.create ~code:E.Storage_busy ~message:(D.message detail) - ~details:[ E.{name=D.detail_name;value=Detail_string (D.encode detail)} ] - |> Result.get_ok + diagnostic_error ~code:E.Storage_busy ~operation:(D.operation detail) + ~reason:(D.reason detail) ~source:(D.encode detail) () ;; @@ -1130,10 +1130,20 @@ let exception_failure ~operation ~reason error = let module D = Logseq_db_types.Failure_diagnostic in let literal = match error with Failure source | Invalid_argument source -> Some source | _ -> None in let detail = failure_detail ?source:literal ~operation ~reason () in - if D.source_class detail <> "unknown" then detail + if Option.is_some (Option.bind literal D.decode) || D.source_class detail <> "unknown" + then detail else failure_detail ~source:(Printexc.to_string error) ~operation ~reason () ;; +let report_worker_error report error = + let module D = Logseq_db_types.Failure_diagnostic in + List.iter (fun (item:Logseq_db_worker_contract.Error.detail) -> + match item.name,item.value with + | name,Detail_string source when name=D.detail_name -> + Option.iter (publish_failure report) (D.decode source) + | _ -> ()) (Logseq_db_worker_contract.Error.details error) +;; + let asset_failure_detail ~operation failure = let reason, source = match failure with | Sync.Asset_network -> "networkFailed", None @@ -1146,13 +1156,17 @@ let asset_failure_detail ~operation failure = | Asset_size_rejected -> "sizeRejected", None | Asset_revoked_access -> "revokedAccess", None | Asset_cancelled -> "cancelled", None - | Asset_invalid_content source -> "invalidContent", Some source in + | Asset_invalid_content source -> "invalidContent", Some source + | Asset_with_cause (_, detail) -> + let module D = Logseq_db_types.Failure_diagnostic in + D.reason detail, Some (D.encode detail) in failure_detail ?source ~operation ~reason () ;; -let report_asset_failure ?detail t scope operation = function +let report_asset_failure ?detail t scope operation failure = + match Sync.asset_failure_kind failure with | Sync.Asset_cancelled -> () - | failure -> + | _ -> let detail = match detail with | None -> asset_failure_detail ~operation failure | Some detail -> @@ -1427,17 +1441,31 @@ let mutation_owner_matches (scope:Sync.graph_scope) intent = ;; let typed_read_error ?code error = - let fail code message = Read.Db_error(Logseq_db_worker_contract.Error.create ~code ~message ~details:[] |> Result.get_ok) in - let safe reason source=Logseq_db_types.Failure_diagnostic.of_source ~operation:"workerOperation" ~reason source - |> Logseq_db_types.Failure_diagnostic.message in + let module E = Logseq_db_worker_contract.Error in + let failure default reason ?source constructor message = + let detail = failure_detail ?source ~operation:"workerOperation" ~reason () in + let module D = Logseq_db_types.Failure_diagnostic in + let message = match source with None -> message | Some _ -> D.message detail in + let origin = E.create_cause ~component:E.Read_model ~operation:"read" + ~code:(Some constructor) ~message |> Result.get_ok in + Read.Db_error (E.create_with_origin ~code:(Option.value code ~default) ~message + ~origin ~details:[E.{name=D.detail_name;value=Detail_string(D.encode detail)}] |> Result.get_ok) in match error with - |Overlay.Stale_read_cursor->(match code with - |None->Read.Stale_read "The read continuation is no longer current." - |Some code->fail code "The read continuation is no longer current.") - |Invalid_read_request source->fail (Option.value code ~default:Logseq_db_worker_contract.Error.Invalid_request) (safe "invalidRequest" source) - |Read_limit_exceeded->fail (Option.value code ~default:Logseq_db_worker_contract.Error.Response_too_large) "The read exceeded its resource limit." - |Database_closed|Snapshot_released|Snapshot_generation_invalidated->fail (Option.value code ~default:Logseq_db_worker_contract.Error.Closed_session) "The read snapshot is no longer available." - |Fatal_read_state source->fail (Option.value code ~default:Logseq_db_worker_contract.Error.Corrupt_storage) (safe "fatalState" source) + | Overlay.Stale_read_cursor -> (match code with + | None -> Read.Stale_read "The read continuation is no longer current." + | Some code -> failure code "stateError" "Stale_read_cursor" "The read continuation is no longer current.") + | Invalid_read_request source -> failure E.Invalid_request "invalidRequest" ~source + "Invalid_read_request" "The read request is invalid." + | Read_limit_exceeded -> failure E.Response_too_large "limitExceeded" + "Read_limit_exceeded" "The read exceeded its resource limit." + | Database_closed -> failure E.Closed_session "databaseClosed" + "Database_closed" "The read database is closed." + | Snapshot_released -> failure E.Closed_session "stateError" + "Snapshot_released" "The read snapshot was released." + | Snapshot_generation_invalidated -> failure E.Closed_session "generationInvalidated" + "Snapshot_generation_invalidated" "The read snapshot generation was invalidated." + | Fatal_read_state source -> failure E.Corrupt_storage "fatalState" ~source + "Fatal_read_state" "The read storage requires recovery." ;; (* A single pinned snapshot supplies each canonical value, including enrichment @@ -1462,7 +1490,7 @@ let read_typed loop limit in let with_snapshot read = match (match snapshot with Some value->Ok value|None->Database.current_snapshot database) with - |Error _->fail Closed_session "The graph snapshot is unavailable." + |Error error->Error(typed_read_error error) |Ok snapshot->Fun.protect ~finally:(fun()->if release_snapshot then Database.release_snapshot snapshot)(fun()-> let exception Enrichment_failed of Read.error in let titles=Hashtbl.create 16 in @@ -1486,7 +1514,8 @@ let read_typed match query with |Read.Changes{generation;after}->fit (fun limit->changes ~generation ~after ~limit) |Admission->checked(match Database.inspect_admission database with - |Error _->fail Closed_session "Admission information is unavailable." + |Error Overlay.Admission_database_closed->Error(typed_read_error Overlay.Database_closed) + |Error (Overlay.Admission_fatal_state source)->Error(typed_read_error (Overlay.Fatal_read_state source)) |Ok value->Ok Worker_pure_data.{active_records=value.active_records;active_bytes=value.active_bytes; protected_wire_bytes=value.protected_wire_bytes;retained_origin_evidence_bytes=value.retained_origin_evidence_bytes; maximum_records=value.maximum_records;maximum_bytes=value.maximum_bytes}) @@ -1736,7 +1765,7 @@ let register_observation | None -> Error "Observation graph is no longer attached" | Some session -> (match Database.listen session.database with - | Error _ -> Error "Observation listener could not register" + | Error error -> Error (Logseq_db_worker_contract.Error.message (listen_error error)) | Ok (subscription, snapshot) -> slot.subscription <- Some subscription; (match sample_observation ~snapshot t slot with @@ -1747,10 +1776,12 @@ let register_observation subscription ~notify:(invalidate_observation t slot) with - | Error _ -> Error "Observation replay could not activate" + | Error error -> Error (Logseq_db_worker_contract.Error.message (listen_error error)) | Ok () -> Ok (0L, response)))) with - | _ -> Error "Observation registration failed"); + | error -> Error (Logseq_db_types.Failure_diagnostic.message + (exception_failure ~operation:"workerOperation" ~reason:"resourceFailed" error))); + !result)))) ;; let refresh_observation t ~registration ~id ~scope = @@ -1876,19 +1907,24 @@ let run_durable_mutation t ~(scope:Sync.graph_scope) ~intent ~current = let existing()=Database.find_local_intent session.database ~namespace:(Mutation.namespace intent) ~mutation_id:(Mutation.mutation_id intent)in let matching prepared=Database.local_intent_payload prepared=payload in + let report error = if not t.stopped && current () then + report_worker_error (report_current_graph t scope) error in + let report_commit error = report (local_commit_error error) in + let report_read error = match typed_read_error error with + | Read.Db_error error -> report error | _ -> () in let reject error=Mutation.Rejected(local_commit_error error)in let prepare()=match existing()with - |Error _->Error Mutation.Unknown + |Error error->report_commit error; Error Mutation.Unknown |Ok(Some prepared)when matching prepared->Ok prepared |Ok(Some _)->Error(reject Overlay.Mutation_identity_conflict) |Ok None-> let append_title=match intent with |Mutation.Append_intent append-> (match Database.current_snapshot session.database with - |Error _->Error Mutation.Unknown + |Error error->report_read error; Error Mutation.Unknown |Ok snapshot->Fun.protect ~finally:(fun()->Database.release_snapshot snapshot)(fun()-> match Database.graph_info snapshot with - |Error _->Error Mutation.Unknown + |Error error->report_read error; Error Mutation.Unknown |Ok info->Ok(Some(journal_append_title info.journal_title_format (Logseq_db_worker_contract.Worker_pure_day.value(Mutation.append_parameters append).day))))) |Mutation.Mutation_intent _->Ok None in @@ -1903,14 +1939,18 @@ let run_durable_mutation t ~(scope:Sync.graph_scope) ~intent ~current = |Error(Overlay.Invalid_local_mutation _ as error) |Error(Overlay.Mutation_identity_conflict as error)-> (match existing()with Ok(Some prepared)when matching prepared->Ok prepared|_->Error(reject error)) - |Error _->Error Mutation.Unknown)in + |Error error->report_commit error; Error Mutation.Unknown)in match prepare()with |Error outcome->outcome |Ok prepared-> if t.stopped || not(current())then Mutation.Unknown else match Database.commit_local_intent session.database prepared with - |Error _->(match Database.inspect_local_intent session.database prepared with - |Ok(Database.Intent_rejected error)->reject error|_->Mutation.Unknown) + |Error error-> + report_commit error; + (match Database.inspect_local_intent session.database prepared with + | Ok(Database.Intent_rejected error)->reject error + | Error error->report_commit error; Mutation.Unknown + | Ok _->Mutation.Unknown) |Ok proof->Mutation.Committed(commit_proof(Database.intent_commit_outcome proof)) ;; @@ -1934,7 +1974,12 @@ let durable_mutation t ~scope ~intent ~current = Fun.protect ~finally:(fun()-> Eio.Semaphore.release t.durable_slots;finish_writer t job) (fun()-> - let outcome=try run_durable_mutation t ~scope ~intent ~current with _->Wire.Unknown in + let outcome=try run_durable_mutation t ~scope ~intent ~current with + | Eio.Cancel.Cancelled _ -> Wire.Unknown + | error -> + if not t.stopped && current () then publish_failure (report_current_graph t scope) + (exception_failure ~operation:"commitLocal" ~reason:"resourceFailed" error); + Wire.Unknown in ignore(Eio.Promise.try_resolve resolve outcome:bool))); submitted:=true); Eio.Promise.await promise) @@ -2340,7 +2385,7 @@ let finish_local_retirement ?(on_diagnostic = fun _ -> ()) t scope deletion capt on_diagnostic (failure_detail ~source ~operation:"deleteMirror" ~reason:"persistenceFailed" ()); source) else Ok () in asset_unit ~on_failure:(fun failure -> - if failure <> Sync.Asset_cancelled then on_diagnostic (asset_failure_detail ~operation:"deleteMirror" failure)) + if Sync.asset_failure_kind failure <> Sync.Asset_cancelled then on_diagnostic (asset_failure_detail ~operation:"deleteMirror" failure)) ~on_diagnostic:(fun detail -> let module D = Logseq_db_types.Failure_diagnostic in on_diagnostic (D.of_source ~operation:"deleteMirror" ~reason:"cleanupFailed" (D.encode detail))) t ~scope @@ -2595,7 +2640,7 @@ let handle_sync_worker_effect t = function let* () = match asset_call t ~scope (Sync.Prepare_graph_asset_deletion retirement) with | Ok (Sync.Asset_deletion_prepared actual) when actual=deletion -> Ok () | Error error -> - if error <> Sync.Asset_cancelled then report (asset_failure_detail ~operation:"deleteMirror" error); + if Sync.asset_failure_kind error <> Sync.Asset_cancelled then report (asset_failure_detail ~operation:"deleteMirror" error); Error (asset_failure_message error) | _ -> report (failure_detail ~source:"Unexpected attachment retirement response." ~operation:"deleteMirror" ~reason:"invalidContent" ()); @@ -2709,8 +2754,19 @@ let handle_sync_worker_effect t = function let owned=Hashtbl.fold (fun (scope:Sync.graph_scope) id result -> if scope.account.user_id=account.user_id && scope.account.managed_sync_origin=account.managed_sync_origin && scope.account.account_generation=account.account_generation then (scope,id)::result else result) t.attachments [] in - List.iter (fun (scope,id)->ignore(close_database_by_id ~scope t id)) owned; - Ok (sync_result ()) + let closed = List.fold_left (fun result (scope,id) -> + let next = try close_database_by_id ~scope t id with + | Eio.Cancel.Cancelled _ as cancelled -> raise cancelled + | error -> + let detail=exception_failure ~operation:"closeDatabase" ~reason:"closeFailed" error in + Error (diagnostic_error ~operation:"closeDatabase" ~reason:"closeFailed" + ~source:(Logseq_db_types.Failure_diagnostic.encode detail) ()) in + (match next with + | Ok () -> () + | Error error -> report_worker_error + (fun signal -> t.dependencies.publish (Core.Diagnostic signal)) error); + match result with Error _ -> result | Ok () -> next) (Ok ()) owned in + Result.map (fun () -> sync_result ()) closed | Inspect_sync scope -> (match attached_session t scope with | None -> Error (effect_error "The overlay database is unavailable.") @@ -2790,8 +2846,7 @@ let handle_sync_worker_effect t = function ()) | Ok (Authoritative_applied commit) -> (match Database.inspect_sync session.database with - | Error _ -> - Error (effect_error "The overlay Sync view is unavailable.") + | Error error -> Error (sync_read_error error) | Ok sync -> Ok (sync_result @@ -2813,15 +2868,39 @@ let run_request t (type a) (request : a Core.runner_request) | Handle_sync_worker_effect worker_effect -> handle_sync_worker_effect t worker_effect ;; +let request_operation : type a. a Core.runner_request -> string = function + | Core.Execute_request _ -> "workerOperation" + | Close_database _ -> "closeDatabase" + | Handle_sync_worker_effect worker_effect -> (match worker_effect with + | Sync.Inspect_mirror _ -> "inspectMirror" | Attach_graph _ -> "attachGraph" + | Inspect_sync _ -> "inspectSync" | Activate_snapshot _ -> "activateSnapshot" + | Apply_authoritative_batch _ -> "applyAuthoritative" + | Apply_outbox_transition _ -> "applyOutbox" | Detach_graph _ -> "closeDatabase" + | Reset_managed_account _ -> "resetAccount" | Delete_mirror _ -> "deleteMirror") +;; + +let request_exception request error = + let operation = request_operation request in + let detail = exception_failure ~operation ~reason:"resourceFailed" error in + diagnostic_error ~operation ~reason:"resourceFailed" + ~source:(Logseq_db_types.Failure_diagnostic.encode detail) () +;; + +let post_worker_result : type a. t -> Core.ticket -> a Core.runner_request + -> (a, Core.effect_error) result -> unit = fun t ticket request result -> + let completion = match request with + | Core.Execute_request _ -> Core.Execute_request_completed (ticket, result) + | Close_database _ -> Core.Close_database_completed (ticket, result) + | Handle_sync_worker_effect _ -> Core.Sync_worker_effect_completed (ticket, result) in + t.post (Core.Runner_completed completion) +;; + let complete t ?writer (Core.Request (ticket, request)) = Fun.protect ~finally:(fun()->Option.iter(finish_writer t)writer)(fun()-> - match request with - |Core.Execute_request _->let result=run_request t request in - t.post(Core.Runner_completed(Core.Execute_request_completed(ticket,result))) - |Close_database _->let result=run_request t request in - t.post(Core.Runner_completed(Core.Close_database_completed(ticket,result))) - |Handle_sync_worker_effect _->let result=run_request t request in - t.post(Core.Runner_completed(Core.Sync_worker_effect_completed(ticket,result)))) + let result = try run_request t request with + | Eio.Cancel.Cancelled _ as cancelled -> raise cancelled + | error -> Error (request_exception request error) in + post_worker_result t ticket request result) ;; let worker_operation_scope t (Core.Request(_,request)) = match request with @@ -2840,19 +2919,32 @@ let worker_operation_scope t (Core.Request(_,request)) = |Reset_managed_account _|Delete_mirror _->None) ;; let submit_worker t (Core.Request(ticket,request) as packed) = - let fork ?writer ()= - try t.dependencies.runtime.fork ~sw:t.sw(fun()->complete t ?writer packed) - with error->Option.iter(finish_writer t)writer;raise error in + let fork ?writer () = + let started = ref false in + try t.dependencies.runtime.fork ~sw:t.sw (fun () -> + started := true; complete t ?writer packed) + with error -> + if !started then raise error; + Option.iter (finish_writer t) writer; + match error with + | Eio.Cancel.Cancelled _ -> raise error + | _ -> post_worker_result t ticket request (Error (request_exception request error)) in match worker_operation_scope t packed with - |None->fork() - |Some(scope,cleanup)->match reserve_writer ~cleanup t scope with - |Ok writer->fork ~writer () - |Error message->let error=effect_error message in - let result=match request with - |Core.Execute_request _->Core.Execute_request_completed(ticket,Error error) - |Close_database _->Core.Close_database_completed(ticket,Error error) - |Handle_sync_worker_effect _->Core.Sync_worker_effect_completed(ticket,Error error) in - t.post(Core.Runner_completed result) + | None -> fork () + | Some(scope,cleanup) -> match reserve_writer ~cleanup t scope with + | Ok writer -> fork ~writer () + | Error message -> post_worker_result t ticket request (Error (effect_error message)) +;; + +let rec upload_failure = function + | Sync.Asset_with_cause (failure, _) -> upload_failure failure + | Asset_network -> Upload.Network + | Asset_authentication | Asset_locked -> Authentication + | Asset_missing_source | Asset_not_found -> Missing_source + | Asset_size_rejected -> Size_rejected + | Asset_revoked_access -> Revoked_access + | Asset_checksum_mismatch | Asset_storage_full | Asset_cancelled + | Asset_invalid_content _ -> Invalid_content ;; let run_upload t (context : Sync.asset_context) instruction = @@ -2974,20 +3066,10 @@ let run_upload t (context : Sync.asset_context) instruction = on_diagnostic ticket (failure_detail ~operation:"assetUpload" ~reason:"invalidContent" ()); Error Upload.Invalid_content | Error failure -> - if current ticket && failure <> Sync.Asset_cancelled then + if current ticket && Sync.asset_failure_kind failure <> Sync.Asset_cancelled then on_diagnostic ticket (Option.value !detail ~default:(asset_failure_detail ~operation:"assetUpload" failure)); - Error - (match failure with - | Sync.Asset_network -> Upload.Network - | Asset_authentication | Asset_locked -> Authentication - | Asset_missing_source | Asset_not_found -> Missing_source - | Asset_size_rejected -> Size_rejected - | Asset_revoked_access -> Revoked_access - | Asset_checksum_mismatch - | Asset_storage_full - | Asset_cancelled - | Asset_invalid_content _ -> Invalid_content)) + Error (upload_failure failure)) Upload.Put_succeeded) | Await_publication (ticket, intent) -> let rec await () = @@ -3039,7 +3121,8 @@ let submit_upload t (context : Sync.asset_context) instruction = with error->Hashtbl.remove t.upload_operations ticket;finish_writer t job;raise error)) ;; -let transfer_failure = function +let rec transfer_failure = function + | Sync.Asset_with_cause (failure, _) -> transfer_failure failure | Sync.Asset_network -> Logseq_sync_pure_reducer.Asset_transfer.Network | Asset_not_found | Asset_missing_source -> Not_found | Asset_checksum_mismatch -> Checksum_mismatch @@ -3097,7 +3180,8 @@ let lookup_local_asset t ~scope ~asset ~checksum ~current ~diagnostic call = in let published_diagnostic = !diagnostic in (match published with - | Error Sync.Asset_storage_full -> Ok (Sync.Asset_cached (Some lease)) + | Error failure when Sync.asset_failure_kind failure = Sync.Asset_storage_full -> + Ok (Sync.Asset_cached (Some lease)) | _ -> (match call (Sync.Release_asset_file lease) with | Error failure when current () -> report_asset_failure t scope "assetRelease" failure @@ -3106,8 +3190,9 @@ let lookup_local_asset t ~scope ~asset ~checksum ~current ~diagnostic call = (match published with | Ok Sync.Asset_unit -> call (Sync.Check_asset_cache (asset, intent.version)) - | Error (Sync.Asset_missing_source | Asset_checksum_mismatch) -> - Ok (Sync.Asset_cached None) + | Error failure when (match Sync.asset_failure_kind failure with + | Sync.Asset_missing_source | Asset_checksum_mismatch -> true + | _ -> false) -> Ok (Sync.Asset_cached None) | Error _ as error -> error | Ok _ -> Error (Sync.Asset_invalid_content "Unexpected publication completion"))) @@ -3136,7 +3221,11 @@ let release_orphan_work t work = let result=try Eio.Cancel.protect(fun()-> release_asset_file_result t ~scope:resource.scope ~handle:resource.handle ~report_failure:(Core.Orphan_cleanup.first_attempt work)) - with _->Error "Attachment cleanup was interrupted." in + with error -> + let detail=exception_failure ~operation:"assetRelease" ~reason:"cleanupFailed" error in + if Core.Orphan_cleanup.first_attempt work then + publish_failure (report_current_graph t resource.scope) detail; + Error (Logseq_db_types.Failure_diagnostic.message detail) in t.orphan_cleanup<-Core.Orphan_cleanup.completed t.orphan_cleanup work result; let key=resource.scope,resource.handle in let waiters=Option.value(Hashtbl.find_opt t.resource_release_waiters key)~default:[] in @@ -3440,11 +3529,15 @@ let run_asset t (context : Sync.asset_context) instruction = | Ok Sync.Asset_retry_elapsed when not t.stopped -> t.post (Core.Asset_completed (context.scope, Transfer.Retry_elapsed id)) | Error Sync.Asset_cancelled -> () - | Ok _ | Error _ -> + | (Ok _ | Error _) as result -> (match t.latest_snapshot with | Some snapshot when not t.stopped && snapshot.startup.account_generation=context.scope.account.account_generation && snapshot.startup.graph_generation=context.scope.graph_generation -> + (match result with + | Error failure -> report_asset_failure t context.scope "assetDownload" failure + | Ok _ -> publish_failure (report_current_graph t context.scope) + (failure_detail ~operation:"assetDownload" ~reason:"invalidContent" ())); t.dependencies.publish (Core.Diagnostic "asset.retryTimer") | _ -> ())) | Notify _ | Backpressure _ | Capacity_available -> () @@ -3688,7 +3781,7 @@ let prepare_import_unlocked with | Ok (Sync.Asset_staged { file; checksum; size }) -> Ok (file, checksum, size) | Error failure -> - if failure <> Sync.Asset_cancelled then + if Sync.asset_failure_kind failure <> Sync.Asset_cancelled then on_diagnostic (Option.value !detail ~default:(asset_failure_detail ~operation:"assetMetadata" failure)); Error (asset_failure_message failure) | Ok _ -> source_error "invalidContent" (Error "Unexpected staging completion") diff --git a/logseq_db_worker/lib/logseq_db_worker.ml b/logseq_db_worker/lib/logseq_db_worker.ml index eb74440b..4514d06c 100644 --- a/logseq_db_worker/lib/logseq_db_worker.ml +++ b/logseq_db_worker/lib/logseq_db_worker.ml @@ -200,8 +200,8 @@ let dispatch t event = when ticket = expected_ticket && actual = expected -> finish result | Some (expected_ticket, _), Pure_reducer.Runner_completed - (Pure_reducer.Sync_worker_effect_completed (ticket, Error _)) - when ticket = expected_ticket ->finish (Error "Mirror deletion failed.") + (Pure_reducer.Sync_worker_effect_completed (ticket, Error error)) + when ticket = expected_ticket ->finish (Error (Logseq_db_worker_contract.Error.message error)) | _ -> ())); List.iter (function |Pure_reducer.Run_worker(Pure_reducer.Request(ticket,Pure_reducer.Handle_sync_worker_effect(Sync.Delete_mirror _))) diff --git a/logseq_db_worker/lui/journal_worker.ml b/logseq_db_worker/lui/journal_worker.ml index 21552809..158108b4 100644 --- a/logseq_db_worker/lui/journal_worker.ml +++ b/logseq_db_worker/lui/journal_worker.ml @@ -912,9 +912,15 @@ let fail_unrecoverable (Packed_client client) error = ;; let exception_message exception_ = - match exception_ with - | Failure message | Invalid_argument message -> message - | _ -> Printexc.to_string exception_ + let module D = Logseq_db_types.Failure_diagnostic in + let source = match exception_ with + | Failure message | Invalid_argument message -> message + | _ -> Printexc.to_string exception_ in + let detail = D.of_source ~operation:"workerOperation" ~reason:"resourceFailed" source in + let detail = if Option.is_some (D.decode source) || D.source_class detail <> "unknown" + then detail else + D.of_source ~operation:"workerOperation" ~reason:"resourceFailed" (Printexc.to_string exception_) in + D.message detail ;; type run_result = diff --git a/logseq_db_worker/lui/journal_worker_runtime.ml b/logseq_db_worker/lui/journal_worker_runtime.ml index e564d28c..2eddb653 100644 --- a/logseq_db_worker/lui/journal_worker_runtime.ml +++ b/logseq_db_worker/lui/journal_worker_runtime.ml @@ -133,11 +133,15 @@ let await_reply (reply : startup_reply) = ;; let set_terminal_from_loop exception_ = - let error = - match exception_ with + let module D = Logseq_db_types.Failure_diagnostic in + let source = match exception_ with | Failure message | Invalid_argument message -> message - | _ -> Printexc.to_string exception_ - in + | _ -> Printexc.to_string exception_ in + let detail = D.of_source ~operation:"workerOperation" ~reason:"resourceFailed" source in + let detail = if Option.is_some (D.decode source) || D.source_class detail <> "unknown" + then detail else + D.of_source ~operation:"workerOperation" ~reason:"resourceFailed" (Printexc.to_string exception_) in + let error = D.message detail in with_control (fun () -> Option.iter (fun client -> Journal_worker.Private.fail_unrecoverable client error) @@ -285,15 +289,17 @@ let ensure_started () = Ok () with | exception_ -> - let error = - match exception_ with - | Failure message | Invalid_argument message -> message - | _ -> Printexc.to_string exception_ - in + let error = match exception_ with + | Unix.Unix_error ((Unix.EACCES | Unix.EPERM), _, _) -> "worker.start.permissionDenied" + | Unix.Unix_error (Unix.ENOSPC, _, _) -> "worker.start.noSpace" + | Invalid_argument _ -> "worker.start.invalidArgument" + | Sys_error _ | Unix.Unix_error _ -> "worker.start.systemError" + | Failure _ -> "worker.start.failure" + | _ -> "worker.start.unknown" in with_control (fun () -> control.state <- Terminal; Condition.broadcast control.condition); - Error ("Failed to spawn OCaml Worker Domain: " ^ error)) + Error error) ;; let fresh_generation () = diff --git a/logseq_db_worker/test/test_effect_runner.ml b/logseq_db_worker/test/test_effect_runner.ml index 1d654680..798faf38 100644 --- a/logseq_db_worker/test/test_effect_runner.ml +++ b/logseq_db_worker/test/test_effect_runner.ml @@ -14,7 +14,7 @@ let account ?(generation = 1) user_id : Sync.account_scope = module Sync_runner = Logseq_sync_effect_runner.Effect_runner -let asset_io_runner ?(fork=(fun ~sw:_ task->task())) ?on_asset_diagnostic ?(token_failure = "test must not use network") ~env ~sw ~support ~post () = +let asset_io_runner ?(pending_budget_bytes=16L) ?(fork=(fun ~sw:_ task->task())) ?on_asset_diagnostic ?(token_failure = "test must not use network") ~env ~sw ~support ~post () = let module R = Sync_runner in let dependencies = R.dependencies @@ -31,7 +31,7 @@ let asset_io_runner ?(fork=(fun ~sw:_ task->task())) ?on_asset_diagnostic ?(toke (R.local_store ~asset_cache_budget_bytes:16L ~asset_maximum_file_bytes:16 - ~asset_pending_budget_bytes:16L + ~asset_pending_budget_bytes:pending_budget_bytes ~application_support_directory:support () |> Result.get_ok) @@ -511,6 +511,56 @@ let asset_sync_config () = let limits=Sync.limits ~maximum_response_bytes:1024 ~maximum_artifact_bytes:1024 ~submission_batch_size:1 |> Result.get_ok in Sync.config ~managed_sync_origin:(Uri.of_string "https://api.logseq.io") ~limits |> Result.get_ok ;; +(* Scheduling failures belong to the effect runner, not the pure reducer. *) +let test_worker_schedule_failure_completes + ?(failure=Unix.Unix_error (Unix.EACCES, "fork", "PRIVATE_SYNTHETIC_PAYLOAD")) + ?(expected_source="io-permission") ?(expected_reason="resourceFailed") () = + T.with_managed (fun fixture -> Eio_main.run (fun _ -> Eio.Switch.run (fun sw -> + let posted = ref [] in + let dependencies = Runner.dependencies + ~runtime:(Runner.runtime ~fork:(fun ~sw:_ _ -> + raise failure) + ~sleep:(fun _ -> ()) |> Result.get_ok) + ~config:fixture.config ~overlay:fixture.overlay + ~sync_runner:(Runner.sync_runner ~retire_account:(fun ~account:_ -> Ok ()) + ~submit:(fun _ -> ()) ~shutdown:(fun () -> ()) ()) + ~publish:(fun _ -> ()) |> Result.get_ok in + let runner = Runner.create ~sw dependencies + ~recovery_current:(fun _ -> false) ~upload_current:(fun _ -> false) + ~asset_current:(fun _ -> false) ~post:(fun event -> posted := event :: !posted) + |> Result.get_ok in + let state = Core.initial (Core.config ~worker:fixture.config ~sync:(asset_sync_config ())) |> Result.get_ok in + let state = (Core.step state (Core.Sync_event (Sync.Sign_in_requested + {account_generation=0;presentation_generation=0}))).next in + let signed_in = Core.step state (Core.Sync_event (Sync.Account_authenticated {user_id=Some "user-1"})) in + let graph : Sync.graph = {graph_id=(Logseq_db_types.Graph_types.Uuid.of_string "94000000-0000-4000-8000-000000000709" |> Result.get_ok);name="fixture"; + schema={major=65;minor=33;exact=true};encrypted=false} in + let catalog = List.find_map (function + | Core.Run_sync (Sync.Request (ticket, Sync.Fetch_catalog _)) -> + Some (Core.step signed_in.next (Core.Sync_event (Sync.Runner_completed + (Sync.Completion (ticket, Ok [graph]))))) + | _ -> None) signed_in.effects |> Option.get in + let selected = Core.step catalog.next (Core.Sync_event (Sync.Graph_selected graph.graph_id)) in + let work = List.find (function Core.Run_worker _ -> true | _ -> false) selected.effects in + Runner.submit runner work; + match !posted with + | [Core.Runner_completed (Core.Sync_worker_effect_completed (_, Error error))] -> + let module E = Logseq_db_worker_contract.Error in + let module D = Logseq_db_types.Failure_diagnostic in + let detail = List.find_map (fun (item:E.detail) -> match item.value with + | E.Detail_string source when item.name=D.detail_name -> D.decode source + | _ -> None) (E.details error) |> Option.get in + Alcotest.(check string) "operation retained" "inspectMirror" (D.operation detail); + Alcotest.(check string) "source cause retained" expected_source (D.source_class detail); + Alcotest.(check string) "specific reason retained" expected_reason (D.reason detail); + let encoded=Yojson.Safe.to_string(E.to_yojson error) in + let marker="PRIVATE_SYNTHETIC_PAYLOAD" in + let rec contains offset = offset + String.length marker <= String.length encoded + && (String.sub encoded offset (String.length marker)=marker || contains (offset+1)) in + Alcotest.(check bool) "private exception argument excluded" false (contains 0) + | _ -> Alcotest.fail "Scheduling failure must post exactly one typed completion"))) +;; + let admitted_asset_state graph_id = let initial=Sync.initial(asset_sync_config()) |> Result.get_ok in let graph : Sync.graph = @@ -570,7 +620,7 @@ type asset_bridge_fixture = ; fixture_asset : Sync.asset_action -> Sync.asset_value } -let with_asset_bridge ?(asynchronous=false) ?(asynchronous_io=false) run = +let with_asset_bridge ?(asynchronous=false) ?(asynchronous_io=false) ?(pending_budget_bytes=16L) run = T.with_managed (fun fixture -> Eio_main.run (fun env -> Eio.Switch.run (fun sw -> @@ -626,7 +676,7 @@ let with_asset_bridge ?(asynchronous=false) ?(asynchronous_io=false) run = in io_runner := Some - (asset_io_runner + (asset_io_runner ~pending_budget_bytes ~fork:(fun ~sw task->if asynchronous_io then Eio.Fiber.fork ~sw task else task()) ~on_asset_diagnostic:(fun ~scope ~operation ~request_operation detail -> Runner.report_asset_diagnostic (Option.get !active_runner) @@ -771,6 +821,87 @@ let picker_import support ordinal : Logseq_db_types.Asset_import.t = } ;; +(* Local staging publication and lease ownership live in the runner, not the + transfer reducer. Exercise real cache pressure and filesystem failures. *) +let test_wrapped_local_asset_fallback mode () = + with_asset_bridge ~asynchronous:true ~pending_budget_bytes:32L (fun fixture -> + let module Transfer=Logseq_sync_pure_reducer.Asset_transfer in + let scope=fixture.context.scope in + let source=picker_import fixture.support 400 in + let prepared=Runner.prepare_import fixture.runner ~context:fixture.context source ~current:(fun()->true) + |> Result.get_ok in + let filler=picker_import fixture.support 410 in + let remote_bytes=String.make 16 'r' in + Out_channel.with_open_bin filler.source_file(fun output->output_string output remote_bytes); + let checksum,file=match fixture.fixture_asset(Sync.Stage_asset_file + {operation=filler.operation;file_type=filler.file_type;source_file=filler.source_file}) with + | Sync.Asset_staged {checksum;file;_}->checksum,file | _->Alcotest.fail "remote cache staging failed" in + let version=Logseq_db_types.Asset_descriptor.version ~checksum ~file_type:"bin" |> Result.get_ok in + ignore(fixture.fixture_asset(Sync.Publish_staged_asset{asset=prepared.asset;version;file})); + let pinned=match fixture.fixture_asset(Sync.Check_asset_cache(prepared.asset,version)) with + | Sync.Asset_cached(Some handle)->handle | _->Alcotest.fail "remote cache did not fill budget" in + Alcotest.(check bool) "original version actually evicted" true + (fixture.fixture_asset(Sync.Check_asset_cache(prepared.asset,prepared.version))=Sync.Asset_cached None); + let descriptor=Logseq_db_types.Asset_descriptor.create ~uuid:prepared.asset + ~source:(Managed(Some version)) ~current_checksum:(Some prepared.version.checksum) + ~size:None ~dimensions:None |> Result.get_ok in + let transfer=Transfer.create(Transfer.config ~active:2 ~foreground_reserved:1 ~pending:3 ~retries:0 |> Result.get_ok) + ~scope ~online:true ~unlocked:true in + let _,work=Transfer.step transfer(Replace{consumer="wrapped-local-fallback";priority=Foreground;assets=[descriptor]}) in + let lookup=List.find_map(function Transfer.Check_cache ticket->Some ticket|_->None)work |> Option.get in + fixture.set_asset_current(fun ticket->ticket=Transfer.Lookup lookup); + let completed,resolve=Eio.Promise.create() in + fixture.on_asset_completed(function Transfer.Cache_checked(_,result)-> + ignore(Eio.Promise.try_resolve resolve result:bool)|_->()); + let publication_failure=ref None and retained_lease=ref None and gated=ref false in + let releases handle=List.fold_left(fun count->function + | Sync.Release_asset_file actual when actual=handle -> count+1 | _->count) 0 !(fixture.actions) in + let remote_checks ()=List.fold_left(fun count->function + | Sync.Check_asset_cache(asset,actual) when asset=prepared.asset && actual=version -> count+1 + | _->count) 0 !(fixture.actions) in + let before_remote=remote_checks () in + let reached=fixture.gate (fun output -> match output.Sync.request.action with + | Sync.Retain_staged_file file when file=prepared.staged_file && not !gated -> + retained_lease:=(match output.result with Ok(Sync.Asset_retained(Some(lease,_)))->Some lease|_->None); + gated:=true;true + | Sync.Publish_staged_asset _ -> + (match output.result with Error failure->publication_failure:=Some failure|_->());false + | _->false) ~deliver_first:false in + Runner.submit fixture.runner(Core.Run_asset(fixture.context,Transfer.Check_cache lookup)); + Eio.Promise.await reached; + let path=Option.get !(fixture.held_path) in + (match mode with + | `Full->() + | `Missing->Unix.unlink path + | `Checksum->Out_channel.with_open_bin path(fun output->output_string output "fake") + | `Oversize->Out_channel.with_open_bin path(fun output->output_string output (String.make 17 'x'))); + fixture.deliver_held ~keep_gate:true (); + let result=Eio.Promise.await completed in + let failure=Option.get !publication_failure in + let expected=match mode with `Full->Sync.Asset_storage_full | `Missing->Asset_missing_source + | `Checksum->Asset_checksum_mismatch | `Oversize->Asset_size_rejected in + Alcotest.(check bool) "actual publication error is wrapped with cause" true + (Sync.asset_failure_kind failure=expected && Option.is_some(Sync.asset_failure_cause failure)); + Alcotest.(check int) "outer staging lease released only on non-full branch" + (if mode=`Full then 0 else 1) (releases (Option.get !retained_lease)); + Alcotest.(check int) "only missing/checksum consult remote cache" + (if mode=`Missing || mode=`Checksum then 1 else 0) (remote_checks () - before_remote); + if mode=`Oversize then + Alcotest.(check bool) "nonfallback failure category reaches transfer unchanged" true + (result=Error(Transfer.Invalid_content "Asset size exceeds the limit")) + else ( + let handle=match result with Ok(Some handle)->handle + | _->Alcotest.fail "wrapped local publication failure broke preview/cache fallback" in + let lease,readable=match fixture.fixture_asset(Sync.Retain_asset_file handle) with + | Sync.Asset_retained(Some file)->file | _->Alcotest.fail "fallback returned an unreadable lease" in + Alcotest.(check string) "fallback owns correct readable bytes" + (if mode=`Full then "file" else remote_bytes) + (In_channel.with_open_bin readable In_channel.input_all); + ignore(fixture.fixture_asset(Sync.Release_asset_file lease)); + ignore(fixture.fixture_asset(Sync.Release_asset_file handle))); + ignore(fixture.fixture_asset(Sync.Release_asset_file pinned))) +;; + let cancelled_stage_output deliver_first () = with_asset_bridge (fun fixture -> let source = picker_import fixture.support 10 in @@ -3104,7 +3235,17 @@ let () = ] ) ; ( "unlock diagnostics", [Alcotest.test_case "typed finite runner observations" `Quick test_unlock_runner_logging] ) ; ( "contract" - , [ Alcotest.test_case "readonly import input recovery and exact proof ACK" `Quick test_import_input_readonly_recovery_and_proof_ack + , [ Alcotest.test_case "wrapped local publication storage full retains staging" `Quick (test_wrapped_local_asset_fallback `Full) + ; Alcotest.test_case "wrapped missing staging falls back to remote cache" `Quick (test_wrapped_local_asset_fallback `Missing) + ; Alcotest.test_case "wrapped checksum failure falls back to remote cache" `Quick (test_wrapped_local_asset_fallback `Checksum) + ; Alcotest.test_case "wrapped nonfallback publication failure releases lease" `Quick (test_wrapped_local_asset_fallback `Oversize) + ; Alcotest.test_case "worker scheduling failure completes" `Quick (fun () -> test_worker_schedule_failure_completes ()) + ; Alcotest.test_case "worker encoded exception preserves reason" `Quick + (test_worker_schedule_failure_completes + ~failure:(Invalid_argument (Logseq_db_storage.Logseq_sqlite_codec.diagnostic_error + (Logseq_db_storage.Logseq_sqlite_codec.Unsupported_tag "PRIVATE_SYNTHETIC_PAYLOAD"))) + ~expected_source:"unknown" ~expected_reason:"codecUnsupportedTag") + ; Alcotest.test_case "readonly import input recovery and exact proof ACK" `Quick test_import_input_readonly_recovery_and_proof_ack ; Alcotest.test_case "initial observation snapshot joins real release" `Quick test_observation_release_waits_for_initial_snapshot ; Alcotest.test_case "host internal followup survives full actor queue" `Quick test_host_followup_survives_full_actor_queue ; Alcotest.test_case "host scope settlement preserves real lease" `Quick test_host_scope_settlement_preserves_live_lease diff --git a/logseq_db_worker/test/test_lui_service.ml b/logseq_db_worker/test/test_lui_service.ml index 150e81a2..9844ffe4 100644 --- a/logseq_db_worker/test/test_lui_service.ml +++ b/logseq_db_worker/test/test_lui_service.ml @@ -609,7 +609,7 @@ type control_request = | Latest of int | Crash of Worker.control_ticket -let control_fixture ?(fail_init = false) () = +let control_fixture ?(fail_init = false) ?(crash = fun () -> failwith "controlled control failure") () = let ticket = Atomic.make None in let replies = Atomic.make 0 and latest = Atomic.make (-1) in @@ -646,7 +646,7 @@ let control_fixture ?(fail_init = false) () = | Latest value -> Atomic.set latest value; Atomic.incr control_count - | Crash _ -> failwith "controlled control failure" + | Crash _ -> crash () | Data _ -> invalid_arg "data is not a short control") ~shutdown:(fun (_, resolve) -> ignore (Eio.Promise.try_resolve resolve () : bool)) () @@ -654,8 +654,8 @@ let control_fixture ?(fail_init = false) () = service, ticket, replies, latest, control_count ;; -let with_control_fixture run = - let service, ticket, replies, latest, control_count = control_fixture () in +let with_control_fixture ?crash run = + let service, ticket, replies, latest, control_count = control_fixture ?crash () in let client = Worker_runtime.start ~runtime_epoch:(ID.Runtime.Epoch.of_int64 2200L) service () |> Result.get_ok @@ -736,6 +736,34 @@ let test_control_client_epoch_and_stop () = "stop has priority over control traffic") ;; +(* The mailbox owns exception-to-terminal conversion; the pure reducer cannot + execute a failing service callback. Use the actual codec diagnostic producer. *) +let test_control_preserves_encoded_failure () = + let module D = Logseq_db_types.Failure_diagnostic in + let module Codec = Logseq_db_storage.Logseq_sqlite_codec in + List.iter (fun (codec_error, related) -> + let detail = Codec.diagnostic_error codec_error |> D.decode |> Option.get in + T.require (D.source_class detail = "unknown") "regression requires a classified reason without a source"; + let detail = if related then D.with_related detail + (D.create ~operation:"closeDatabase" ~reason:"closeFailed" ~source:"database is locked" ()) + else detail in + with_control_fixture ~crash:(fun () -> invalid_arg (D.encode detail)) + (fun client ticket _ _ _ -> + T.require (Worker.send_control client (Crash ticket) = Worker.Control_accepted) + "encoded failure callback accepted"; + let terminal = ref false in + control_wait "encoded failure reaches terminal" (fun () -> + List.iter (function + | Worker.Terminal {error;_} -> + T.require (error = D.message detail) + "terminal must retain codec reason and related failure despite unknown source"; + terminal := true + | _ -> ()) (Worker.For_testing.drain_events client ~max_events:64); + !terminal))) + [Codec.Unsupported_tag "PRIVATE_SYNTHETIC_TAG", false; + Codec.Malformed_transit "PRIVATE_SYNTHETIC_PAYLOAD", true] +;; + let test_control_cancel_and_failure () = with_control_fixture (fun client ticket _ _ _ -> let cancelled = Worker.send client (Data 1) |> accepted in @@ -772,7 +800,10 @@ let test_control_cancel_and_failure () = control_wait "control exception reaches terminal" (fun () -> List.iter (function - | Worker.Terminal _ -> terminal := true + | Worker.Terminal {error;_} -> + T.require (error = "workerOperation: Resource could not be opened. Cause: An operation raised Failure.") + "control exception must preserve its safe category without its raw payload"; + terminal := true | _ -> ()) (Worker.For_testing.drain_events client ~max_events:64); !terminal); @@ -1883,6 +1914,7 @@ let () = ; T.case "control capabilities bind client and stop" test_control_client_epoch_and_stop + ; T.case "control encoded codec exception retains reason" test_control_preserves_encoded_failure ; T.case "control cancellation startup and exceptions" test_control_cancel_and_failure ; T.case "bounded control pressure and data fairness" diff --git a/logseq_overlay_db/lib/mirror.ml b/logseq_overlay_db/lib/mirror.ml index a651129b..a93b374a 100644 --- a/logseq_overlay_db/lib/mirror.ml +++ b/logseq_overlay_db/lib/mirror.ml @@ -172,9 +172,9 @@ let close_storage staged = | Ok () -> staged.storage_closed <- true; Ok () - | Error _ -> + | Error error -> Printf.eprintf "journal: phase=cleanup operation=closeSnapshotStorage code=closePending recovery=reopen\n%!"; - Error "snapshot storage cannot be closed") + Error (Storage.diagnostic_error ~operation:"activateSnapshot" error)) ;; let initialize_plaintext_table sqlite = diff --git a/logseq_overlay_db/test/test_overlay_storage.ml b/logseq_overlay_db/test/test_overlay_storage.ml index bf15ac00..a34eeac0 100644 --- a/logseq_overlay_db/test/test_overlay_storage.ml +++ b/logseq_overlay_db/test/test_overlay_storage.ml @@ -3386,10 +3386,229 @@ let sqlite_full_int64_checkpoint () = "SQLite update preserves the complete cursor") ;; + +let storage_corruption_after_open operation () = + let module Storage = Logseq_db_storage.Logseq_sqlite_storage in + let module D = Logseq_db_types.Failure_diagnostic in + T.with_temp_directory "storage-read-cause-" (fun directory -> + let path = Filename.concat directory "mirror.sqlite" in + T.copy_storage_fixture path; + let connection = Storage.open_database path |> Result.get_ok in + let callbacks = Storage.connection_callbacks connection in + Fun.protect ~finally:(fun () -> ignore (Storage.close callbacks)) (fun () -> + let sqlite = Sqlite3.db_open path in + Sqlite3.exec sqlite "UPDATE kvs SET content = '{invalid' WHERE addr = 0" |> Sqlite3.Rc.check; + T.require (Sqlite3.db_close sqlite) "close corrupting writer"; + let result = match operation with + | `Startup -> Result.map (fun _ -> ()) (Storage.startup_metadata connection) + | `Garbage -> Result.map (fun _ -> ()) (Storage.garbage_stats callbacks) in + match result with + | Ok () -> Alcotest.fail "corruption was ignored" + | Error error -> + let diagnostic = Storage.diagnostic_error ~operation:"attachGraph" error |> D.decode |> Option.get in + Alcotest.(check string) "read returns concrete typed cause" + "codecMalformedTransit" (D.reason diagnostic))) +;; + +(* Commit_batch owns the sequence of persistence stages; no reducer executes + its callbacks. Keep both the failing stage and the concrete SQLite cause. *) +let storage_commit_retains_stage stage expected_reason () = + let module Storage = Logseq_db_storage.Logseq_sqlite_storage in + let module D = Logseq_db_types.Failure_diagnostic in + T.with_temp_directory "storage-commit-cause-" (fun directory -> + let path = Filename.concat directory "mirror.sqlite" in + T.copy_storage_fixture path; + let connection = Storage.open_database path |> Result.get_ok in + let base = Storage.connection_callbacks connection in + Fun.protect ~finally:(fun () -> ignore (Storage.close base)) (fun () -> + let fail _ = Error "SQLITE_FULL" in + let callbacks = match stage with + | `Metadata -> { base with upsert_sync_metadata = fail } + | `Outbox -> { base with replace_sync_outbox = fail } + | `Mutation -> { base with upsert_mutation_receipts = fail } + | `Terminal -> { base with upsert_terminal_batch_receipts = fail } in + let metadata = Logseq_db_types.Sync_checkpoint.create + ~graph_id:T.graph_uuid ~schema:Logseq_db_types.Graph_types.{ major=1; minor=0 } + ~applied_server_t:(server_cursor_of_int 0) ~checksum:"0000000000000000" + |> Result.get_ok in + let batch = Storage.{ writes=[]; + sync_metadata=(if stage=`Metadata then Some metadata else None); + sync_outbox=(if stage=`Outbox then Some [] else None); + mutation_receipts=(if stage=`Mutation then Some [] else None); + terminal_batch_receipts=(if stage=`Terminal then Some [] else None) } in + match Storage.commit_batch callbacks batch with + | Ok () -> Alcotest.fail "failed persistence stage was ignored" + | Error error -> + let diagnostic = Storage.diagnostic_error ~operation:"commitLocal" error + |> D.decode |> Option.get in + Alcotest.(check string) "failing commit stage" expected_reason (D.reason diagnostic); + Alcotest.(check string) "underlying SQLite cause" "sqlite-full" (D.source_class diagnostic))) +;; + +let codec_preserves_structural_cause () = + let module D = Logseq_db_types.Failure_diagnostic in + match Codec.decode_storage_payload "{}" with + | Ok _ -> Alcotest.fail "unknown storage map was accepted" + | Error error -> + let diagnostic = Codec.diagnostic_error error |> D.decode |> Option.get in + Alcotest.(check string) "concrete structural cause" "storage-payload-map-shape" + (D.source_class diagnostic) +;; + +let storage_rollback_retains_primary () = + let module Storage = Logseq_db_storage.Logseq_sqlite_storage in + let module D = Logseq_db_types.Failure_diagnostic in + T.with_temp_directory "storage-rollback-cause-" (fun directory -> + let path = Filename.concat directory "mirror.sqlite" in + T.copy_storage_fixture path; + let connection = Storage.open_database path |> Result.get_ok in + let base = Storage.connection_callbacks connection in + Fun.protect ~finally:(fun () -> ignore (Storage.close base)) (fun () -> + let callbacks = { base with + upsert = (fun _ -> Error "SQLITE_FULL"); + rollback = (fun () -> base.rollback (); failwith "SQLITE_BUSY") } in + let batch = Storage.{ writes=[{address="0";payload="";addresses=[]}]; + sync_metadata=None;sync_outbox=None;mutation_receipts=None;terminal_batch_receipts=None } in + match Storage.commit_batch callbacks batch with + | Error (Storage.Write_failed _ as error) -> + let diagnostic = Storage.diagnostic_error ~operation:"commitLocal" error |> D.decode |> Option.get in + Alcotest.(check string) "primary write cause" "sqlite-full" (D.source_class diagnostic); + let secondary = D.related diagnostic |> Option.get in + Alcotest.(check string) "rollback stage" "storageRollbackFailed" (D.reason secondary); + Alcotest.(check string) "rollback cause" "sqlite-locked" (D.source_class secondary) + | _ -> Alcotest.fail "rollback replaced the primary typed write error")) +;; + +let related_diagnostic_validation () = + let module D = Logseq_db_types.Failure_diagnostic in + let primary = D.create ~operation:"closeDatabase" ~reason:"storageCheckpointFailed" + ~source:"SQLITE_FULL" ~cursor:Int64.max_int ~http_status:599 () in + let secondary = D.create ~operation:"closeDatabase" ~reason:"storageCloseFailed" + ~source:"SQLITE_BUSY" ~cursor:Int64.max_int ~http_status:599 () in + let encoded = D.with_related primary secondary |> D.encode in + T.require (String.length encoded <= 1024) "dual diagnostic exceeds wire budget"; + T.require (String.length (D.message (D.with_related primary secondary)) <= 512) + "dual diagnostic exceeds display budget"; + let prefix = "journal-failure:v1:" in + let fields = match Yojson.Safe.from_string + (String.sub encoded (String.length prefix) (String.length encoded - String.length prefix)) with + | `Assoc fields -> fields | _ -> assert false in + let reject fields = + T.require (D.decode (prefix ^ Yojson.Safe.to_string (`Assoc fields)) = None) + "unsafe related diagnostic was accepted" in + reject (("related", `String (D.encode secondary)) :: fields); + reject (("unexpected", `String "SECRET") :: fields); + reject (("related", `String "SECRET") :: List.remove_assoc "related" fields); + reject (("related", `String encoded) :: List.remove_assoc "related" fields) +;; + +let diagnostic_unix_cause error expected () = + let module D = Logseq_db_types.Failure_diagnostic in + let source = Printexc.to_string (Unix.Unix_error (error, "SECRET-operation", "SECRET-path")) in + let diagnostic = D.of_source ~operation:"workerOperation" ~reason:"persistenceFailed" source in + Alcotest.(check string) "concrete errno" expected (D.source_class diagnostic); + let encoded = D.encode diagnostic in + Alcotest.(check bool) "no exception argument" false (String.contains encoded 'S') +;; + +(* Session owns finalization of both callbacks; reducer events cannot execute + checkpoint and close. Inject distinct callback failures at that public owner. *) +let storage_close_retains_both_causes ~checkpoint_raises () = + let module Storage = Logseq_db_storage.Logseq_sqlite_storage in + let module Session = Logseq_db_storage.Storage_session in + let module D = Logseq_db_types.Failure_diagnostic in + T.with_temp_directory "storage-close-causes-" (fun directory -> + let path = Filename.concat directory "mirror.sqlite" in + T.copy_storage_fixture path; + let connection = Storage.open_database path |> Result.get_ok in + let base = Storage.connection_callbacks connection in + let callbacks = { base with + checkpoint = (fun () -> if checkpoint_raises then failwith "SQLITE_FULL" else Error "SQLITE_FULL"); + close = (fun () -> ignore (Storage.close base); Error "SQLITE_BUSY") } in + let session = Session.create ~tail:[] ~callbacks in + match Session.close session with + | Error (Session.Persistence_failed encoded) -> + let diagnostic = match D.decode encoded with + | Some value -> value + | None -> Alcotest.fail "dual failure destroyed the structured carrier" in + Alcotest.(check string) "checkpoint stage" "storageCheckpointFailed" (D.reason diagnostic); + Alcotest.(check string) "checkpoint cause" "sqlite-full" (D.source_class diagnostic); + let secondary = D.related diagnostic |> Option.get in + Alcotest.(check string) "close stage" "storageCloseFailed" (D.reason secondary); + Alcotest.(check string) "close cause" "sqlite-locked" (D.source_class secondary); + let wrapped = D.of_source ~operation:"attachGraph" ~reason:"resourceFailed" encoded in + Alcotest.(check string) "upper wrapper retains secondary" "storageCloseFailed" + (D.reason (D.related wrapped |> Option.get)); + let message = D.message diagnostic in + let contains text = + let rec loop n = n + String.length text <= String.length message + && (String.sub message n (String.length text) = text || loop (n + 1)) in + loop 0 in + T.require (contains "close" && contains "locked") "close cause was discarded" + | _ -> Alcotest.fail "close errors were lost") +;; + +(* Corruption is produced by the SQLite/Transit adapter, not a pure reducer. + Exercise its public open boundary with real malformed persisted bytes. *) +let storage_open_preserves_codec_cause content expected_reason () = + let module Storage = Logseq_db_storage.Logseq_sqlite_storage in + let module D = Logseq_db_types.Failure_diagnostic in + T.with_temp_directory "storage-cause-" (fun directory -> + let path = Filename.concat directory "mirror.sqlite" in + T.copy_storage_fixture path; + let sqlite = Sqlite3.db_open path in + let statement = Sqlite3.prepare sqlite "UPDATE kvs SET content = ? WHERE addr = 0" in + Sqlite3.Rc.check (Sqlite3.bind_text statement 1 content); + Sqlite3.Rc.check (Sqlite3.step statement); + Sqlite3.Rc.check (Sqlite3.finalize statement); + T.require (Sqlite3.db_close sqlite) "close corrupted fixture"; + match Storage.open_database path with + | Ok connection -> + ignore (Storage.close (Storage.connection_callbacks connection)); + Alcotest.fail "corrupt root was accepted" + | Error error -> + let encoded = Storage.diagnostic_error ~operation:"attachGraph" error in + let diagnostic = D.decode encoded |> Option.get in + Alcotest.(check string) "concrete codec cause survives storage boundary" + expected_reason (D.reason diagnostic); + let rec contains_secret offset = + offset + 6 <= String.length encoded + && (String.sub encoded offset 6 = "SECRET" || contains_secret (offset + 1)) + in + T.require (not (contains_secret 0)) + "diagnostic must not include synthetic secret") +;; + let () = Alcotest.run "logseq_overlay_db storage" - [ ( "legacy status compatibility" + [ ( "storage error causes" + , [ Alcotest.test_case "codec structural cause" `Quick codec_preserves_structural_cause + ; Alcotest.test_case "rollback retains primary cause" `Quick storage_rollback_retains_primary + ; Alcotest.test_case "related diagnostic validates and bounds input" `Quick related_diagnostic_validation + ; Alcotest.test_case "startup corruption returns result" `Quick + (storage_corruption_after_open `Startup) + ; Alcotest.test_case "garbage traversal corruption returns result" `Quick + (storage_corruption_after_open `Garbage) + ; Alcotest.test_case "metadata commit stage" `Quick + (storage_commit_retains_stage `Metadata "storageMetadataFailed") + ; Alcotest.test_case "outbox commit stage" `Quick + (storage_commit_retains_stage `Outbox "storageOutboxFailed") + ; Alcotest.test_case "mutation receipt commit stage" `Quick + (storage_commit_retains_stage `Mutation "storageMutationReceiptFailed") + ; Alcotest.test_case "terminal receipt commit stage" `Quick + (storage_commit_retains_stage `Terminal "storageTerminalReceiptFailed") + ; Alcotest.test_case "Unix I/O failure" `Quick (diagnostic_unix_cause Unix.EIO "io-error") + ; Alcotest.test_case "Unix connection reset" `Quick (diagnostic_unix_cause Unix.ECONNRESET "network-reset") + ; Alcotest.test_case "close retains both failures" `Quick (storage_close_retains_both_causes ~checkpoint_raises:false) + ; Alcotest.test_case "throwing checkpoint still closes" `Quick + (storage_close_retains_both_causes ~checkpoint_raises:true) + ; Alcotest.test_case "malformed Transit survives open" `Quick + (storage_open_preserves_codec_cause "{SECRET-synthetic-invalid" "codecMalformedTransit") + ; Alcotest.test_case "unsupported tag survives open" `Quick + (storage_open_preserves_codec_cause {|["~#SECRET-synthetic-tag","SECRET"]|} "codecUnsupportedTag") + ] ) + ; ( "legacy status compatibility" , [ Alcotest.test_case "format 17 status and dependency shadows" `Quick diff --git a/logseq_sync/lib/effect_runner/effect_runner.ml b/logseq_sync/lib/effect_runner/effect_runner.ml index d36c0907..7f710b5f 100644 --- a/logseq_sync/lib/effect_runner/effect_runner.ml +++ b/logseq_sync/lib/effect_runner/effect_runner.ml @@ -631,7 +631,8 @@ let encrypt_protected_values t handle plaintexts = | plaintext :: rest -> (match t.dependencies.crypto.encrypt_aes_gcm ~key:graph_key ~plaintext with | Ok value -> encrypt (value :: encrypted) rest - | Error message -> Error message) + | Error source -> Error Failure_diagnostic.( + of_source ~operation:"encryptValue" ~reason:"cryptoInvalid" source |> encode)) in encrypt [] plaintexts ;; @@ -1055,6 +1056,17 @@ let network_request : type a. a Core.runner_request -> bool = function | Encrypt_protected_values _ | Decrypt_protected_values _ -> false ;; +let request_diagnostic_operation : type a. a Core.runner_request -> string = function + | Core.Fetch_catalog _ | Load_catalog _ -> "loadCatalog" + | Save_catalog _ -> "saveCatalog" + | Fetch_snapshot_baseline _ | Fetch_snapshot_metadata _ | Download_snapshot _ -> "decodeSnapshot" + | Delete_account_secrets _ -> "resetAccount" + | Encrypt_protected_values _ -> "encryptValue" + | Decrypt_protected_values _ -> "decryptValue" + | Fetch_e2ee_graph_key _ | Fetch_e2ee_user_keys _ | Load_and_unlock_graph_key _ + | Fetch_and_unlock_graph_key _ | Unlock_private_key _ -> "platformCrypto" +;; + let submit_request : type a. t -> a Core.effect_ticket -> a Core.runner_request -> unit = fun t ticket request -> let id = Core.effect_ticket_id ticket |> Core.effect_id_to_string in @@ -1092,6 +1104,10 @@ let submit_request : type a. t -> a Core.effect_ticket -> a Core.runner_request raise Runner_cancelled)) with | Runner_cancelled -> None + | Eio.Cancel.Cancelled _ as error -> raise error + | error -> Some (Error (Core.Effect_failed Failure_diagnostic.( + of_source ~operation:(request_diagnostic_operation request) ~reason:"unavailable" + (Printexc.to_string error) |> encode))) in Hashtbl.remove t.operations id; Option.iter (fun kind -> @@ -1368,15 +1384,24 @@ let asset_diagnostic_operation = function | _ -> "assetPreview" ;; +let asset_exception (request : Core.asset_io_request) error = + Core.Asset_invalid_content Failure_diagnostic.( + of_source ~operation:(asset_diagnostic_operation request.action) ~reason:"invalidContent" + (Printexc.to_string error) |> encode) +;; + let with_asset_diagnostics t (request : Core.asset_io_request) ~current work = let operation = asset_diagnostic_operation request.action in let captured = ref None in let report ~reason:reason_name source = captured := Some (Failure_diagnostic.of_source ~operation ~reason:reason_name source) in - let result = Eio.Fiber.with_binding asset_diagnostic_capture report work in - (match result with - | Error Core.Asset_cancelled | Ok _ -> () - | Error failure when not t.closed && current () -> + let result = Eio.Fiber.with_binding asset_diagnostic_capture report (fun () -> + try work () with + | (Runner_cancelled | Eio.Cancel.Cancelled _) as error -> raise error + | error -> Error (asset_exception request error)) in + match result with + | Error Core.Asset_cancelled | Ok _ -> result + | Error failure -> let fallback = match failure with | Core.Asset_network -> "networkFailed", "" | Asset_not_found -> "notFound", "" @@ -1388,15 +1413,16 @@ let with_asset_diagnostics t (request : Core.asset_io_request) ~current work = | Asset_size_rejected -> "sizeRejected", "" | Asset_revoked_access -> "revokedAccess", "" | Asset_invalid_content source -> "invalidContent", source - | Asset_cancelled -> "cancelled", "" in + | Asset_cancelled -> "cancelled", "" + | Asset_with_cause (_, cause) -> Failure_diagnostic.reason cause, Failure_diagnostic.encode cause in let reason_name, source = fallback in let detail = Option.value !captured ~default:( Failure_diagnostic.of_source ~operation ~reason:reason_name source) in (* Diagnostics must never change an operation's completion or cleanup. *) - (try t.on_asset_diagnostic ~scope:request.context.scope ~operation - ~request_operation:request.operation detail with _ -> ()) - | Error _ -> ()); - result + if not t.closed && current () then + (try t.on_asset_diagnostic ~scope:request.context.scope ~operation + ~request_operation:request.operation detail with _ -> ()); + Error (Core.asset_failure_with_cause failure detail) ;; type asset_encryption = @@ -2215,7 +2241,8 @@ let submit_asset_io t instruction ticket (request : Core.asset_io_request) = raise Runner_cancelled) with | Runner_cancelled -> Error Core.Asset_cancelled - | error -> Error (Core.Asset_invalid_content (Printexc.to_string error))) + | Eio.Cancel.Cancelled _ as error -> raise error + | error -> Error (asset_exception request error)) in (* A completed resource must reach the reducer even after cancellation. The reducer owns acceptance and issues cleanup for stale resources. *) @@ -2273,7 +2300,11 @@ let submit_protected t instruction ticket (request : Core.protected_request) = with | Runner_cancelled -> Error (Core.Effect_failed "Protected request cancelled") - | error -> Error (Core.Effect_failed (Printexc.to_string error))) + | Eio.Cancel.Cancelled _ as error -> raise error + | error -> Error (Core.Effect_failed Failure_diagnostic.( + of_source ~operation:(match request.protected_action with + | Core.Encrypt_values _ -> "encryptValue" | Decrypt_value _ -> "decryptValue") + ~reason:"cryptoInvalid" (Printexc.to_string error) |> encode))) in t.post (Core.Runner_completed (Core.Protected_completion (ticket, result))))) ;; @@ -2338,7 +2369,8 @@ let submit t instruction = ~current:(fun () -> not t.closed) (fun () -> execute_asset t request ~current:(fun () -> false))) with - | error -> Error (Core.Asset_invalid_content (Printexc.to_string error)) + | Eio.Cancel.Cancelled _ as error -> raise error + | error -> Error (asset_exception request error) in t.post (Core.Runner_completed (Core.Asset_completion (ticket, result))))) else if t.closed diff --git a/logseq_sync/lib/effect_runner/eio/http_eio.ml b/logseq_sync/lib/effect_runner/eio/http_eio.ml index 0ff3efdd..42692f32 100644 --- a/logseq_sync/lib/effect_runner/eio/http_eio.ml +++ b/logseq_sync/lib/effect_runner/eio/http_eio.ml @@ -23,9 +23,7 @@ let exception_message = function | Transport_error message -> message | End_of_file -> "premature HTTP EOF" | Eio.Time.Timeout -> "HTTP transport timed out" - | Tls_eio.Tls_alert _ | Tls_eio.Tls_failure _ -> "TLS transport terminated" - | Sys_error _ | Unix.Unix_error _ -> "transport file or socket operation failed" - | _ -> "HTTP transport failed" + | error -> Tls_client_eio.transport_failure ~operation:"httpTransport" error ;; let tls_stream_socket flow = diff --git a/logseq_sync/lib/effect_runner/eio/tls_client_eio.ml b/logseq_sync/lib/effect_runner/eio/tls_client_eio.ml index d184c708..0928ff15 100644 --- a/logseq_sync/lib/effect_runner/eio/tls_client_eio.ml +++ b/logseq_sync/lib/effect_runner/eio/tls_client_eio.ml @@ -193,3 +193,25 @@ let connect_diagnosed ~on_diagnostic ~sw ~authenticator ~network ~clock ~host ~p let connect ~sw ~authenticator ~network ~clock ~host ~port = connect_diagnosed ~on_diagnostic:(fun _ -> ()) ~sw ~authenticator ~network ~clock ~host ~port ;; + +(* Only finite TLS alert constructors and classified exception identities cross + the transport boundary; certificate names, payloads and socket paths do not. *) +let transport_failure ~operation:operation_name error = + let alert_name = function + | Tls.Packet.UNKNOWN _ -> "unknown" + | alert -> Tls.Packet.alert_type_to_string alert in + let source = match error with + | Tls_eio.Tls_alert alert -> "TLS alert: " ^ alert_name alert + | Tls_eio.Tls_failure failure -> + let _, alert = Tls.Engine.alert_of_failure failure in + "TLS failure: " ^ alert_name alert + | Eio.Io (Eio.Net.E (Eio.Net.Connection_failure (Eio.Net.Refused _)), _) -> + "Unix.Unix_error(Unix.ECONNREFUSED,,)" + | Eio.Io (Eio.Net.E (Eio.Net.Connection_failure Eio.Net.Timeout), _) -> + "Unix.Unix_error(Unix.ETIMEDOUT,,)" + | Eio.Io (Eio.Net.E (Eio.Net.Connection_reset _), _) -> + "Unix.Unix_error(Unix.ECONNRESET,,)" + | error -> Printexc.to_string error in + Logseq_db_types.Failure_diagnostic.( + of_source ~operation:operation_name ~reason:"networkFailed" source |> encode) +;; diff --git a/logseq_sync/lib/effect_runner/eio/tls_client_eio.mli b/logseq_sync/lib/effect_runner/eio/tls_client_eio.mli index 68b39c12..168782d2 100644 --- a/logseq_sync/lib/effect_runner/eio/tls_client_eio.mli +++ b/logseq_sync/lib/effect_runner/eio/tls_client_eio.mli @@ -46,3 +46,6 @@ val connect_diagnosed_with_attention -> host:string -> port:int -> (Tls_eio.t, error) result + +(** Bounded safe carrier retaining errno or finite TLS alert category. *) +val transport_failure : operation:string -> exn -> string diff --git a/logseq_sync/lib/effect_runner/eio/websocket_eio.ml b/logseq_sync/lib/effect_runner/eio/websocket_eio.ml index 3e112916..730f4c8c 100644 --- a/logseq_sync/lib/effect_runner/eio/websocket_eio.ml +++ b/logseq_sync/lib/effect_runner/eio/websocket_eio.ml @@ -74,10 +74,11 @@ let peer_termination payload = if String.length payload < 2 then Core.Recoverable_close { message = None; attention = Requires_attention } else ( - match (Char.code payload.[0] lsl 8) lor Char.code payload.[1] with - | 1002 | 1003 | 1007 | 1008 | 1009 | 1010 -> - Core.Protocol_failure "WebSocket peer rejected the protocol or policy" - | _ -> Core.Recoverable_close { message = None; attention = Requires_attention }) + let status = (Char.code payload.[0] lsl 8) lor Char.code payload.[1] in + let message = Printf.sprintf "WebSocket peer closed with status %d" status in + match status with + | 1002 | 1003 | 1007 | 1008 | 1009 | 1010 -> Core.Protocol_failure message + | _ -> Core.Recoverable_close { message = Some message; attention = Requires_attention }) ;; let closed_reason t = @@ -437,8 +438,7 @@ let failure_message = function | Protocol_error message -> message | End_of_file -> "WebSocket peer EOF" | Eio.Time.Timeout -> "WebSocket establishment timed out" - | Tls_eio.Tls_alert _ | Tls_eio.Tls_failure _ -> "WebSocket TLS termination" - | _ -> "WebSocket transport failed" + | error -> Tls_client_eio.transport_failure ~operation:"websocketTransport" error ;; let failure_reason = function diff --git a/logseq_sync/lib/effect_runner/protocol/catalog.ml b/logseq_sync/lib/effect_runner/protocol/catalog.ml index 9b78b5ec..e3374ae9 100644 --- a/logseq_sync/lib/effect_runner/protocol/catalog.ml +++ b/logseq_sync/lib/effect_runner/protocol/catalog.ml @@ -56,12 +56,26 @@ type graph_dto = type response = { graphs : Yojson.Safe.t list } [@@deriving of_yojson { strict = false }] +let decode_fields decoder json = + let fields = List.map (fun (field, wire) -> field, + "field " ^ wire ^ " is missing or has an invalid type") + [ "Catalog.response.graphs", "graphs" + ; "Catalog.readiness.ready", "graph-ready-for-use?" + ; "Catalog.graph_dto.graph_id", "graph-id" + ; "Catalog.graph_dto.graph_name", "graph-name" + ; "Catalog.graph_dto.schema_version", "schema-version" + ; "Catalog.graph_dto.encrypted", "graph-e2ee?" ] in + Logseq_db_types.Json_codec.decode decoder json + |> Result.map_error (fun message -> + "graph catalog: " ^ Logseq_db_types.Json_codec.error_context ~fields message) +;; + let decode_graph json = - bind (readiness_of_yojson json) (fun ready -> + bind (decode_fields readiness_of_yojson json) (fun ready -> match ready.ready with | None | Some false -> Ok None | Some true -> - bind (graph_dto_of_yojson json) (fun dto -> + bind (decode_fields graph_dto_of_yojson json) (fun dto -> bind (bounded_text ~maximum:512 "graph name" dto.graph_name) (fun name -> bind (schema dto.schema_version) (fun schema -> Ok @@ -76,7 +90,7 @@ let decode_graph json = let decode source = try bind - (Logseq_db_types.Json_codec.decode + (decode_fields response_of_yojson (Yojson.Safe.from_string source)) (fun response -> diff --git a/logseq_sync/lib/effect_runner/protocol/e2ee.ml b/logseq_sync/lib/effect_runner/protocol/e2ee.ml index 82750cd7..0ee069cd 100644 --- a/logseq_sync/lib/effect_runner/protocol/e2ee.ml +++ b/logseq_sync/lib/effect_runner/protocol/e2ee.ml @@ -63,7 +63,11 @@ let graph_key_response source = (Logseq_db_types.Json_codec.decode graph_key_response_dto_of_yojson (Yojson.Safe.from_string source) - |> Result.map_error (fun _ -> "E2EE graph key response shape is invalid")) + |> Result.map_error (fun message -> + let fields = List.map (fun (field, wire) -> "E2ee.graph_key_response_dto." ^ field, + "field " ^ wire ^ " is missing or has an invalid type") + ["encrypted_aes_key", "encrypted-aes-key"; "encrypted_aes_key_base64", "encrypted-aes-key-base64"] in + "E2EE graph key response: " ^ Logseq_db_types.Json_codec.error_context ~fields message)) (fun dto -> (* The optional compatibility field is validated by PPX but not consumed. *) let _ = dto.encrypted_aes_key_base64 in @@ -78,7 +82,11 @@ let user_keys_response source = (Logseq_db_types.Json_codec.decode user_keys_response_dto_of_yojson (Yojson.Safe.from_string source) - |> Result.map_error (fun _ -> "E2EE user keys response shape is invalid")) + |> Result.map_error (fun message -> + let fields = List.map (fun (field, wire) -> "E2ee.user_keys_response_dto." ^ field, + "field " ^ wire ^ " is missing or has an invalid type") + ["public_key", "public-key"; "encrypted_private_key", "encrypted-private-key"] in + "E2EE user keys response: " ^ Logseq_db_types.Json_codec.error_context ~fields message)) (fun dto -> bind (bounded_string "public-key" dto.public_key) (fun _ -> bounded_string "encrypted-private-key" dto.encrypted_private_key)) diff --git a/logseq_sync/lib/pure_reducer/core.ml b/logseq_sync/lib/pure_reducer/core.ml index 6fef459f..6e35db66 100644 --- a/logseq_sync/lib/pure_reducer/core.ml +++ b/logseq_sync/lib/pure_reducer/core.ml @@ -340,7 +340,14 @@ let decode_catalog_cache source = catalog_cache_of_yojson (Yojson.Safe.from_string source) with - | Error _ -> Error "unsupported or invalid catalog cache format" + | Error message -> + let fields = List.concat_map (fun (prefix, names) -> + List.map (fun name -> prefix ^ name, + "field " ^ name ^ " is missing or has an invalid type") names) + [ "Core.catalog_cache.", ["cache_user_id"; "cache_graphs"; "cache_selected_graph"; "cache_resident_graphs"] + ; "Core.graph.", ["graph_id"; "name"; "schema"; "encrypted"] + ; "Core.cache_schema.", ["major"; "minor"; "exact"] ] in + Error ("catalog cache: " ^ Logseq_db_types.Json_codec.error_context ~fields message) | Ok cache -> if (not (bounded_text cache.cache_user_id)) @@ -361,8 +368,8 @@ let decode_catalog_cache source = then Error "catalog cache contains invalid domain values" else Ok cache with - | Yojson.Json_error _ | Failure _ | Invalid_argument _ -> - Error "unsupported or invalid catalog cache format" + | Yojson.Json_error _ -> Error "catalog cache: invalid JSON syntax" + | Failure _ | Invalid_argument _ -> Error "catalog cache: invalid shape" ;; type encryption_batch = @@ -460,6 +467,21 @@ type asset_failure = | Asset_revoked_access | Asset_cancelled | Asset_invalid_content of string + | Asset_with_cause of asset_failure * Logseq_db_types.Failure_diagnostic.t + +let rec asset_failure_kind = function + | Asset_with_cause (failure, _) -> asset_failure_kind failure + | failure -> failure +;; + +let asset_failure_cause = function + | Asset_with_cause (_, cause) -> Some cause + | _ -> None +;; + +let asset_failure_with_cause failure cause = + Asset_with_cause (asset_failure_kind failure, cause) +;; type asset_value = | Asset_unit @@ -1323,6 +1345,16 @@ let start_websocket core graph = { next; effects = [ publish next; Run (Start_websocket { scope; uri }) ] } ;; +let authoritative_input_error_message = function + | Overlay.Invalid_server_cursor -> "authoritative server cursor is invalid" + | Invalid_encoded_transaction -> "authoritative transaction is empty or its byte limit is invalid" + | Authoritative_transaction_too_large -> "authoritative transaction exceeds the byte limit" + | Authoritative_batch_empty -> "authoritative batch contains no transactions" + | Authoritative_batch_too_large -> "authoritative batch exceeds its transaction count or byte limit" + | Authoritative_cursor_not_strictly_ordered -> "authoritative transaction cursors are not strictly ordered" + | Authoritative_through_mismatch -> "authoritative batch final cursor does not match its through cursor" +;; + let checksum value = Overlay.Checksum.of_string ("checksum:v1:" ^ value) let decode_snapshot_baseline source = @@ -1353,7 +1385,11 @@ let decode_snapshot_uri source = | Some "https", Some host, None, None when String.length host > 0 -> Ok (uri, server_t) | _ -> Error "snapshot download URL is invalid")) - | Ok _ | Error _ -> Error "snapshot metadata is invalid" + | Ok _ -> Error "snapshot metadata: server reported ok=false" + | Error message -> + let fields = List.map (fun name -> "Core.snapshot_uri_response." ^ name, + "field " ^ name ^ " is missing or has an invalid type") ["ok"; "url"; "t"] in + Error ("snapshot metadata: " ^ Logseq_db_types.Json_codec.error_context ~fields message) with | Yojson.Json_error _ -> Error "snapshot metadata is not valid JSON" ;; @@ -2191,7 +2227,7 @@ let authoritative_input core t checksum_value txs = ~maximum_bytes:core.config.limits.maximum_response_bytes tx.Sync_protocol.Server.tx with - | Error _ -> Error "invalid encoded transaction" + | Error failure -> Error (authoritative_input_error_message failure) | Ok transaction -> encode (Overlay.authoritative_transaction ~cursor:tx.t ~transaction :: acc) @@ -2211,7 +2247,7 @@ let authoritative_input core t checksum_value txs = ~through:t ~checksum |> Result.map Option.some - |> Result.map_error (fun _ -> "invalid authoritative batch")))) + |> Result.map_error authoritative_input_error_message))) ;; let enqueue_authoritative core batch = @@ -2751,7 +2787,25 @@ let complete_catalog_load core cache = else restored ;; -let deletion_failed core stage = +let deletion_failed ?source core stage = + let core = match source with + | None -> core + | Some source -> + let operation, reason = match stage with + | Closing_graph -> "closeDatabase", "closeFailed" + | Deleting_mirror -> "deleteMirror", "deleteFailed" + | Clearing_selection -> "saveCatalog", "persistenceFailed" in + let diagnostic = Logseq_db_types.Failure_diagnostic.of_source + ~operation ~reason source in + let groups = core.public_state.diagnostics.groups in + let entries = List.concat_map (fun group -> + if group.title = "Sync errors" then group.entries else []) groups in + let group = { title = "Sync errors"; entries = + (operation, Logseq_db_types.Failure_diagnostic.message diagnostic) + :: List.remove_assoc operation entries } in + { core with public_state = { core.public_state with diagnostics = + { groups = group :: List.filter (fun group -> group.title <> "Sync errors") groups } } } + in let message = match stage with | Closing_graph -> "Local graph close failed." @@ -3071,7 +3125,7 @@ let graph_detached core scope result = match core.deletion_scope, core.public_state.snapshot.local_deletion with | Some captured, Some (Deletion_in_progress Closing_graph) when captured = scope -> (match result with - | Error _ -> deletion_failed core Closing_graph + | Error source -> deletion_failed ~source core Closing_graph | Ok () -> let next = set_snapshot @@ -3101,7 +3155,7 @@ let mirror_deleted core (request : mirror_deletion) result = && request.scope = effect_scope_of_graph scope -> (match result with | Error message -> - let failed = deletion_failed core Deleting_mirror in + let failed = deletion_failed ~source:message core Deleting_mirror in if List.mem message [ "Switch to another graph before deleting." ; "The local copy is still in use. Close attachment previews and try again." ] diff --git a/logseq_sync/spec/pure_reducer/core.mli b/logseq_sync/spec/pure_reducer/core.mli index 4e4e2711..696d6441 100644 --- a/logseq_sync/spec/pure_reducer/core.mli +++ b/logseq_sync/spec/pure_reducer/core.mli @@ -339,6 +339,12 @@ type asset_failure = | Asset_revoked_access | Asset_cancelled | Asset_invalid_content of string + | Asset_with_cause of asset_failure * Logseq_db_types.Failure_diagnostic.t + +(** Preserve the retry category independently of bounded diagnostic context. *) +val asset_failure_kind : asset_failure -> asset_failure +val asset_failure_cause : asset_failure -> Logseq_db_types.Failure_diagnostic.t option +val asset_failure_with_cause : asset_failure -> Logseq_db_types.Failure_diagnostic.t -> asset_failure type asset_value = | Asset_unit diff --git a/logseq_sync/test/asset_adapter_memory.ml b/logseq_sync/test/asset_adapter_memory.ml index 8a6764cc..b06727c9 100644 --- a/logseq_sync/test/asset_adapter_memory.ml +++ b/logseq_sync/test/asset_adapter_memory.ml @@ -242,7 +242,7 @@ let () = |> Option.get in let allocated = Gc.allocated_bytes () -. before in - if outcome <> Error C.Asset_authentication || !auth_calls <> 1 + if Result.map_error C.asset_failure_kind outcome <> Error C.Asset_authentication || !auth_calls <> 1 then failwith "production adapter did not encode the maximum-size asset"; Printf.printf "Production upload OCaml allocation bytes: %.0f\n%!" allocated; if allocated > 128. *. 1024. *. 1024. diff --git a/logseq_sync/test/core_contract.ml b/logseq_sync/test/core_contract.ml index 4d1de7ee..dceb8070 100644 --- a/logseq_sync/test/core_contract.ml +++ b/logseq_sync/test/core_contract.ml @@ -6631,3 +6631,93 @@ let resident_graph_failure_preserves_account_catalog_request () = ;; let scenarios=scenarios @ [Alcotest.test_case "resident graph failure preserves account catalog ticket" `Quick resident_graph_failure_preserves_account_catalog_request] ;; + +(* The reducer owns deletion failure projection; valid producer diagnostics must + survive its public completion boundary without exposing arbitrary source text. *) +let local_deletion_preserves_cause () = + let module D = Logseq_db_types.Failure_diagnostic in + List.iter (fun source -> + let original, _, _, _ = deletion_ready_graph false in + let ready, _, _, _ = deletion_noncurrent original in + let deleting = Core.step ready (Core.Local_cache_deletion_requested graph_id) in + let request = deletion_mirror deleting.effects in + let event = Core.Mirror_deleted (request, Error source) in + let failed = Core.step deleting.next event in + ignore (check_replay "deletion-cause" deleting.next event failed); + let diagnostics = (Core.state failed.next).diagnostics.groups in + let diagnostic = List.find_map (fun (group : Core.diagnostic_group) -> + List.assoc_opt "deleteMirror" group.entries) diagnostics in + let expected = D.of_source ~operation:"deleteMirror" ~reason:"deleteFailed" source in + Alcotest.(check (option string)) "deletion publishes the bounded concrete cause" + (Some (D.message expected)) diagnostic; + check_no_work failed.next [event]) + [ D.create ~operation:"deleteMirror" ~reason:"persistenceFailed" + ~source:"Unix.Unix_error(Unix.ENOSPC,PRIVATE_PATH,PRIVATE_PATH)" () |> D.encode + ; D.create ~operation:"deleteMirror" ~reason:"ownershipConflict" () |> D.encode + ; "Bearer PRIVATE_TOKEN https://user:PRIVATE_PASSWORD@host/PRIVATE_GRAPH" + ] +;; +let scenarios = scenarios @ [Alcotest.test_case "local deletion preserves concrete safe cause" + `Quick local_deletion_preserves_cause] +;; + +let authoritative_input_retains_validation_cause () = + let transaction t tx = Protocol.Server.{ t = server_cursor_of_int t; tx; outliner_op = None } in + List.iter (fun (txs, through, expected) -> + let opened, connection, _ = pull_fixture () in + let event = Core.Websocket_message (connection, Protocol.Server.Pull_ok + { t = server_cursor_of_int through; checksum = None; txs }) in + let failed = Core.step opened.next event in + ignore (check_replay "authoritative-input-cause" opened.next event failed); + Alcotest.(check (option string)) "typed validation cause survives to caller" + (Some expected) (Core.state failed.next).snapshot.last_error) + [ [transaction 1 ""], 1, "authoritative transaction is empty or its byte limit is invalid" + ; [transaction 1 (String.make (Logseq_db_types.Limits.maximum_response_bytes + 1) 'x')], 1, + "authoritative transaction exceeds the byte limit" + ; [transaction 1 "[]"; transaction 1 "[]"], 1, + "authoritative transaction cursors are not strictly ordered" + ; [transaction 2 "[]"; transaction 1 "[]"], 1, + "authoritative transaction cursors are not strictly ordered" + ; [transaction 1 "[]"], 2, "authoritative batch final cursor does not match its through cursor" + ; List.init 4097 (fun i -> transaction (i + 1) "[]"), 4097, + "authoritative batch exceeds its transaction count or byte limit" + ] +;; +let scenarios = scenarios @ [Alcotest.test_case "authoritative input retains validation cause" + `Quick authoritative_input_retains_validation_cause] +;; + +let cache_decoder_keeps_safe_field_cause () = + List.iter (fun ((source : string), expected) -> + Alcotest.(check (option string)) "cache parser retains field cause without data" + (Some expected) + (match Core.decode_catalog_cache source with Error message -> Some message | Ok _ -> None)) + [ {|{"cache_user_id":false,"cache_graphs":[],"cache_selected_graph":null}|}, + "catalog cache: field cache_user_id is missing or has an invalid type" + ; {|{"cache_user_id":"PRIVATE_USER","cache_graphs":false,"cache_selected_graph":null}|}, + "catalog cache: field cache_graphs is missing or has an invalid type" + ; {|{"cache_user_id":"PRIVATE_USER","cache_user_id":"PRIVATE_USER2","cache_graphs":[],"cache_selected_graph":null}|}, + "catalog cache: duplicate JSON field" + ] +;; +let snapshot_metadata_keeps_safe_field_cause () = + List.iter (fun ((source : string), expected) -> + let requested, _ = begin_snapshot_bootstrap () in + let failed = List.find_map (function + | Core.Run (Core.Request (ticket, Core.Fetch_snapshot_metadata _)) -> + Some (Core.step requested.next (Core.Runner_completed (Core.Completion (ticket, Ok source)))) + | _ -> None) requested.effects |> Option.get in + Alcotest.(check (option string)) "metadata parser publishes field cause without URL" + (Some expected) (Core.state failed.next).snapshot.last_error) + [ {|{"ok":true,"t":"PRIVATE_BODY","url":"https://user:PRIVATE_PASSWORD@example.com"}|}, + "snapshot metadata: field t is missing or has an invalid type" + ; {|{"ok":true,"t":1,"url":false}|}, + "snapshot metadata: field url is missing or has an invalid type" + ; {|{"ok":false,"t":1,"url":"https://example.com"}|}, + "snapshot metadata: server reported ok=false" + ] +;; +let scenarios = scenarios @ [ + Alcotest.test_case "cache decoder keeps safe field cause" `Quick cache_decoder_keeps_safe_field_cause; + Alcotest.test_case "snapshot metadata keeps safe field cause" `Quick snapshot_metadata_keeps_safe_field_cause] +;; diff --git a/logseq_sync/test/runner_contract.ml b/logseq_sync/test/runner_contract.ml index d42309e0..28c762f3 100644 --- a/logseq_sync/test/runner_contract.ml +++ b/logseq_sync/test/runner_contract.ml @@ -412,7 +412,7 @@ let protected_run runner ~tasks ~posted core operation scope action = ;; let test_cached_wrapped_key_is_unlocked_before_protected_value_decryption - ?(malformed_plaintext = false) ?(decryption_failure = false) ?(cancel_online = false) () = + ?(malformed_plaintext = false) ?(decryption_failure = false) ?(decryption_exception = false) ?(encryption_failure = false) ?(cancel_online = false) () = with_support (fun support -> Eio_main.run (fun environment -> Eio.Switch.run (fun sw -> @@ -449,7 +449,8 @@ let test_cached_wrapped_key_is_unlocked_before_protected_value_decryption Error "encryption was not expected") ~decrypt_aes_gcm:(fun ~key ~iv:_ ~ciphertext:_ -> if String.equal key raw_key - then if decryption_failure + then if decryption_exception then raise (Unix.Unix_error (Unix.EACCES, "PRIVATE_OPERATION", "PRIVATE_PATH")) + else if decryption_failure then Error "SENSITIVE-synthetic-crypto-error" else if malformed_plaintext then Ok {|"~XSENSITIVE-synthetic-body"|} @@ -526,8 +527,29 @@ let test_cached_wrapped_key_is_unlocked_before_protected_value_decryption unlocked "decrypt-cached" (Core.graph_key_handle_scope handle) - (Core.Decrypt_value (handle, ciphertext)) in - if decryption_failure then + (if encryption_failure then Core.Encrypt_values (handle, ["plaintext"]) + else Core.Decrypt_value (handle, ciphertext)) in + if encryption_failure then + Alcotest.(check bool) "encryption failure returns a bounded diagnostic" true + (match result with + | Error (Core.Effect_failed source) -> + let module D = Logseq_db_types.Failure_diagnostic in + (match D.decode source with + | Some detail -> D.operation detail = "encryptValue" + && D.reason detail = "cryptoInvalid" + | None -> false) + | _ -> false) + else if decryption_exception then + Alcotest.(check bool) "protected provider exception retains only its safe cause" true + (match result with + | Error (Core.Effect_failed source) -> + let module D = Logseq_db_types.Failure_diagnostic in + (match D.decode source with + | Some detail -> D.operation detail = "decryptValue" + && D.reason detail = "cryptoInvalid" && D.source_class detail = "io-permission" + | None -> false) + | _ -> false) + else if decryption_failure then Alcotest.(check bool) "crypto error excludes sensitive detail" true (match result with | Error (Core.Effect_failed message) -> @@ -1391,7 +1413,7 @@ let scenarios = ;; (* Migrated asset cache public runner scenarios. *) -let with_local_asset_cache ?(budget = 8L) ?(pending_budget = 8L) +let with_local_asset_cache ?(budget = 8L) ?(pending_budget = 8L) ?(with_causes = false) ?on_asset_diagnostic ?(acquire = fun _ -> Ok "token") test = with_support (fun support -> Eio_main.run (fun environment -> @@ -1477,7 +1499,8 @@ let with_local_asset_cache ?(budget = 8L) ?(pending_budget = 8L) in state := transition.next; match pump (consume [] transition.effects) with - | [ output ] -> output.Core.result + | [ output ] -> if with_causes then output.Core.result + else Result.map_error Core.asset_failure_kind output.Core.result | _ -> fail "cache operation did not resolve through reducer output" in test support request)))) @@ -2437,14 +2460,21 @@ let test_asset_authentication_cause_is_scoped () = let reports = ref [] in let on_asset_diagnostic ~scope ~operation ~request_operation detail = reports := (scope, operation, request_operation, detail) :: !reports in - with_local_asset_cache ~on_asset_diagnostic + with_local_asset_cache ~with_causes:true ~on_asset_diagnostic ~acquire:(fun _ -> Error "SENSITIVE-fixture-token-error") (fun _ request -> let version = Logseq_db_types.Asset_descriptor.version ~checksum:(String.make 64 'a') ~file_type:"bin" |> Result.get_ok in + let result = request (Core.Fetch_asset {asset=graph_id (); version; maximum_plaintext_bytes=4}) in Alcotest.(check bool) "original authentication category" true - (request (Core.Fetch_asset {asset=graph_id (); version; maximum_plaintext_bytes=4}) - = Error Core.Asset_authentication); + (Result.map_error Core.asset_failure_kind result = Error Core.Asset_authentication); + (match result with + | Error failure -> + let cause = Core.asset_failure_cause failure in + Alcotest.(check (option string)) "result carries cause without side-channel dependency" + (Some "authenticationFailed") + (Option.map Logseq_db_types.Failure_diagnostic.reason cause) + | Ok _ -> fail "expected authentication failure"); match !reports with | [(scope, operation, request_operation, detail)] -> let module D = Logseq_db_types.Failure_diagnostic in @@ -2500,3 +2530,77 @@ let test_resident_websocket_owners_are_independent () = ;; let scenarios=scenarios @ [Alcotest.test_case "resident physical sockets admit and retire independently" `Quick test_resident_websocket_owners_are_independent] ;; + +(* Provider exceptions occur in the effect runner, not the pure reducer. Reproduce + through its public dependencies and completion output, never a fabricated result. *) +let test_asset_provider_exception_keeps_safe_cause () = + let module D = Logseq_db_types.Failure_diagnostic in + List.iter (fun error -> + let reports = ref [] in + let expected = D.of_source ~operation:"assetDownload" ~reason:"invalidContent" + (Printexc.to_string error) in + with_local_asset_cache ~with_causes:true + ~on_asset_diagnostic:(fun ~scope:_ ~operation:_ ~request_operation:_ detail -> + reports := detail :: !reports) + ~acquire:(fun _ -> raise error) + (fun _ request -> + let version = Logseq_db_types.Asset_descriptor.version + ~checksum:(String.make 64 'a') ~file_type:"bin" |> Result.get_ok in + (match request (Core.Fetch_asset {asset=graph_id (); version; maximum_plaintext_bytes=4}) with + | Error failure -> + let source = match Core.asset_failure_kind failure with + | Core.Asset_invalid_content source -> source + | _ -> fail "expected invalid content exception category" in + Alcotest.(check (option string)) "exception reaches caller as safe bounded cause" + (Some (D.message expected)) (Option.map D.message (D.decode source)) + | _ -> fail "expected classified provider exception"); + Alcotest.(check (list string)) "exception also reaches scoped diagnostics" + [D.message expected] (List.map D.message !reports))) + [ Unix.Unix_error (Unix.EACCES, "PRIVATE_OPERATION", "PRIVATE_PATH") + ; Unix.Unix_error (Unix.ENOSPC, "PRIVATE_OPERATION", "PRIVATE_PATH") + ; Failure "Bearer PRIVATE_TOKEN https://user:PRIVATE_PASSWORD@host/PRIVATE_GRAPH" ] +;; +let scenarios = scenarios @ [Alcotest.test_case + "asset provider exception keeps safe cause" `Quick test_asset_provider_exception_keeps_safe_cause] +;; + +let test_catalog_provider_exception_completes () = + with_support (fun support -> Eio_main.run (fun environment -> Eio.Switch.run (fun sw -> + let tasks = Queue.create () and posted = ref [] in + let deps = dependencies ~environment ~support + ~fork:(fun ~sw:_ task -> Queue.add task tasks) + ~id_token_dependency:(Runner.id_token_provider + ~acquire:(fun _ -> raise (Unix.Unix_error (Unix.EACCES, "PRIVATE_OPERATION", "PRIVATE_PATH"))) + ~invalidate:(fun _ ~token:_ -> ())) () in + let runner = Runner.create ~sw deps ~post:(fun event -> posted := event :: !posted) + |> Result.get_ok in + let requested = Core.step (core ()) (Core.Account_authenticated {user_id=Some "user-1"}) in + let runnable = List.find_map (function + | Core.Run (Core.Request (_, Core.Fetch_catalog _) as runnable) -> Some runnable + | _ -> None) requested.effects |> Option.get in + Runner.submit runner runnable; + while not (Queue.is_empty tasks) do (Queue.take tasks) () done; + (match !posted with + | [Core.Runner_completed (Core.Completion (_, Error (Core.Effect_failed source)))] -> + let module D = Logseq_db_types.Failure_diagnostic in + let diagnostic = D.decode source |> Option.get in + Alcotest.(check string) "provider exception retains errno" "io-permission" (D.source_class diagnostic); + let failed = Core.step requested.next (List.hd !posted) in + Alcotest.(check bool) "pending caller completes with explicit failure" true + ((Core.state failed.next).snapshot.sync_phase = Core.Failed) + | _ -> fail "provider exception must complete the caller"); + Runner.shutdown runner))) +;; +let scenarios = scenarios @ [Alcotest.test_case + "catalog provider exception completes" `Quick test_catalog_provider_exception_completes] +;; + +let scenarios = scenarios @ [Alcotest.test_case + "protected provider exception keeps safe cause" `Quick + (test_cached_wrapped_key_is_unlocked_before_protected_value_decryption ~decryption_exception:true)] +;; + +let scenarios = scenarios @ [Alcotest.test_case + "protected encryption failure keeps safe cause" `Quick + (test_cached_wrapped_key_is_unlocked_before_protected_value_decryption ~encryption_failure:true)] +;; diff --git a/logseq_sync/test/transport_contract.ml b/logseq_sync/test/transport_contract.ml index bc1f5bab..af178aa3 100644 --- a/logseq_sync/test/transport_contract.ml +++ b/logseq_sync/test/transport_contract.ml @@ -95,6 +95,7 @@ try: try: conn.sendall(wire[offset:offset+step]) except (ssl.SSLError, OSError): break if step<32: time.sleep(.0005) + if case.get('corrupt_tls'): os.write(conn.fileno(), b'\x17\x03\x03\x00\x20' + bytes(32)) with open(root+'/sent','w') as f: f.write('sent') if case.get('eof'): if case.get('reset'): conn.setsockopt(socket.SOL_SOCKET,socket.SO_LINGER,struct.pack('ii',1,0)) @@ -432,7 +433,7 @@ let asset_collect t core posted = collect core [] posted ;; -let asset_run t ~clock ~posted core scope operation action = +let asset_run ?(with_cause = false) t ~clock ~posted core scope operation action = let core = asset_current_core t core scope in posted := []; let transition = Core.step core (Core.Asset_requested { scope; operation; action }) in @@ -460,7 +461,8 @@ let asset_run t ~clock ~posted core scope operation action = outputs) in match outputs with - | [ output ] -> output.Core.result + | [ output ] -> if with_cause then output.Core.result + else Result.map_error Core.asset_failure_kind output.Core.result | _ -> Alcotest.fail "asset request did not publish exactly one typed outcome" ;; @@ -1968,7 +1970,7 @@ let asset_upload ~status ~refresh () = let clock = Eio.Stdenv.clock environment in let file = asset_stage runner ~clock ~posted selected.next scope source_file in let result = - asset_run + asset_run ~with_cause:true runner ~clock ~posted @@ -1985,7 +1987,7 @@ let asset_upload ~status ~refresh () = | 413 -> Error Core.Asset_size_rejected | _ -> Error Core.Asset_network in - Alcotest.(check bool) "explicit upload result" true (result = expected); + Alcotest.(check bool) "explicit upload result" true (Result.map_error Core.asset_failure_kind result = expected); (match status, !diagnostics with | 200, [] -> () | 200, _ -> Alcotest.fail "successful retry must not emit a failure diagnostic" @@ -1993,6 +1995,11 @@ let asset_upload ~status ~refresh () = Alcotest.(check string) "finite diagnostic operation" "assetUpload" operation; Alcotest.(check string) "exact request identity" "upload" request_operation; let module D = Logseq_db_types.Failure_diagnostic in + let result_cause = match result with + | Error failure -> Core.asset_failure_cause failure + | Ok _ -> None in + Alcotest.(check (option string)) "typed completion preserves exact diagnostic" + (Some (D.encode detail)) (Option.map D.encode result_cause); Alcotest.(check bool) "safe numeric HTTP status survives" true (String.ends_with ~suffix:(Printf.sprintf "(http=%d)." status) (D.message detail)); Alcotest.(check bool) "source status preserved by strict carrier" true @@ -3428,7 +3435,7 @@ type socket_peaks = ; mutable attempts : int } -let metered_network ?(fail_first = false) network peaks = +let metered_network ?(fail_first = false) ?(before_read = fun () -> ()) network peaks = let module Socket = struct type t = { flow : [ `Generic | `Unix ] Eio.Net.stream_socket_ty Eio.Resource.t @@ -3441,7 +3448,11 @@ let metered_network ?(fail_first = false) network peaks = ] let read_methods = [] - let single_read t = Eio.Flow.single_read t.flow + let single_read t buffer = + before_read (); + let count = Eio.Flow.single_read t.flow buffer in + before_read (); + count let single_write t = Eio.Flow.single_write t.flow let copy t ~src = Eio.Flow.copy src t.flow let shutdown t = Eio.Flow.shutdown t.flow @@ -4203,7 +4214,7 @@ let scenarios = scenarios @ ;; -let test_external_http_contract kind ~success body () = +let test_external_http_contract ?expected_error kind ~success body () = ignore (with_peer [ case (unlock_json_wire body) ] @@ -4270,7 +4281,13 @@ let test_external_http_contract kind ~success body () = | Core.Runner_completed (Core.Completion (_, result)) -> Result.is_ok result | _ -> false) - !posted))) + !posted); + Option.iter (fun expected -> + let actual = List.find_map (function + | Core.Runner_completed (Core.Completion (_, Error (Core.Effect_failed message | Core.Crypto_failed (_, message)))) -> Some message + | _ -> None) !posted in + Alcotest.(check (option string)) "safe decoder field cause reaches caller" + (Some expected) actual) expected_error)) : Yojson.Basic.t) ;; @@ -4798,3 +4815,121 @@ let scenarios=scenarios @ [Alcotest.test_case "resident real TLS channels keep hidden pull and retirement independent" `Quick Resident_network_fixture.run] ;; + +(* The peer close status is owned by frame transport; a reducer receives only + the already-decoded termination and cannot reproduce loss of this status. *) +let test_websocket_close_retains_status () = + List.iter (fun code -> + let payload = String.init 2 (function + | 0 -> Char.chr (code lsr 8) | _ -> Char.chr (code land 255)) + ^ "PRIVATE_GRAPH_CONTENT" in + let report = with_peer [case ~ws:true (frame 8 payload)] + (fun ~environment ~sw ~support ~origin -> + let posted = ref [] and invalidations = ref 0 in + let t = runner ~environment ~sw ~support ~posted ~invalidations () in + Runner.submit t (websocket_start origin); + let closed () = List.find_map (function + | Core.Websocket_closed (_, reason) -> Some reason | _ -> None) !posted in + wait (Eio.Stdenv.clock environment) (fun () -> Option.is_some (closed ())); + let message = match Option.get (closed ()) with + | Core.Protocol_failure message -> Some message + | Core.Recoverable_close {message; _} -> message + | _ -> None in + Alcotest.(check (option string)) "close code survives without peer reason text" + (Some (Printf.sprintf "WebSocket peer closed with status %d" code)) message; + Runner.shutdown t) in + check_retired report) [1000;1008;1011] +;; +let scenarios = scenarios @ [Alcotest.test_case + "WebSocket peer close retains status" `Quick test_websocket_close_retains_status] +;; + +(* The fault occurs after TLS setup in the real transport owner. A reducer only + receives its completed error and cannot reproduce loss of errno/TLS status. *) +let test_transport_runtime_cause ~websocket error expected () = + let armed = ref false and injected = ref false in + let peaks = {sockets=0;socket_peak=0;dials=0;dial_peak=0;owner_peak=0;attempts=0} in + let wire = if websocket then frame 9 "probe" + else response ~headers:"Content-Length: 8192\r\n" (String.make 8192 'x') in + let config = case ~ws:websocket ~segment:16 wire in + let config = if websocket then match config with + | `Assoc fields -> `Assoc (("frame_delay", `Float 0.05) :: fields) | _ -> config + else config in + let report = with_peer [config] + (fun ~environment ~sw ~support ~origin -> + let network = metered_network (Eio.Stdenv.net environment) peaks + ~before_read:(fun () -> if !armed then (injected := true; raise error)) in + let posted = ref [] and invalidations = ref 0 in + let t = runner ~network ~environment ~sw ~support ~posted ~invalidations + ~on_post:(function + | Core.Websocket_opened _ when websocket -> armed := true + | Core.Snapshot_download_progress {received_bytes; _} + when not websocket && received_bytes > 0L -> armed := true + | _ -> ()) () in + Runner.submit t (if websocket then websocket_start origin else operation ~download:true origin); + let failure () = List.find_map (function + | Core.Websocket_closed (_, Core.Recoverable_close {message; _}) -> message + | Core.Runner_completed (Core.Completion (_, Error (Core.Effect_failed message))) -> Some message + | _ -> None) !posted in + wait (Eio.Stdenv.clock environment) (fun () -> Option.is_some (failure ())); + Alcotest.(check bool) "failure occurred after transport setup" true !injected; + let module D = Logseq_db_types.Failure_diagnostic in + let source = Option.get (failure ()) in + let actual = Option.map D.message (D.decode source) in + let operation = if websocket then "websocketTransport" else "httpTransport" in + let expected = D.of_source ~operation ~reason:"networkFailed" expected in + Alcotest.(check (option string)) "transport retains bounded concrete cause" + (Some (D.message expected)) actual; + Runner.shutdown t) in + check_retired report +;; +let scenarios = scenarios @ List.concat_map (fun websocket -> + let prefix = if websocket then "WebSocket" else "HTTP" in + [ Alcotest.test_case (prefix ^ " runtime preserves errno") `Quick + (test_transport_runtime_cause ~websocket + (Unix.Unix_error (Unix.EACCES, "PRIVATE_OPERATION", "PRIVATE_PATH")) + "Unix.Unix_error(Unix.EACCES,PRIVATE_OPERATION,PRIVATE_PATH)") + ]) [false;true] +;; + +let test_transport_tls_cause websocket () = + let config = match case ~ws:websocket "" with + | `Assoc fields -> `Assoc (("corrupt_tls", `Bool true) :: fields) + | _ -> assert false in + let report = with_peer [config] (fun ~environment ~sw ~support ~origin -> + let posted = ref [] and invalidations = ref 0 in + let t = runner ~environment ~sw ~support ~posted ~invalidations () in + Runner.submit t (if websocket then websocket_start origin else operation ~download:false origin); + let failure () = List.find_map (function + | Core.Websocket_closed (_, Core.Recoverable_close {message; _}) -> message + | Core.Runner_completed (Core.Completion (_, Error (Core.Effect_failed message))) -> Some message + | _ -> None) !posted in + wait (Eio.Stdenv.clock environment) (fun () -> Option.is_some (failure ())); + let module D = Logseq_db_types.Failure_diagnostic in + let actual = Option.map D.message (D.decode (Option.get (failure ()))) in + let operation = if websocket then "websocketTransport" else "httpTransport" in + let expected = D.of_source ~operation ~reason:"networkFailed" "TLS failure: bad record mac" in + Alcotest.(check (option string)) "TLS record authentication cause survives transport" + (Some (D.message expected)) actual; + Runner.shutdown t) in + check_retired report +;; +let scenarios = scenarios @ [ + Alcotest.test_case "HTTP runtime preserves TLS record cause" `Quick (test_transport_tls_cause false); + Alcotest.test_case "WebSocket runtime preserves TLS record cause" `Quick (test_transport_tls_cause true)] +;; + +let scenarios = scenarios @ [ + Alcotest.test_case "graph key decoder preserves field cause" `Quick + (test_external_http_contract `Graph_key ~success:false + ~expected_error:"E2EE graph key response: field encrypted-aes-key is missing or has an invalid type" + {|{"encrypted-aes-key":false,"PRIVATE_FIELD":"PRIVATE_CONTENT"}|}); + Alcotest.test_case "user keys decoder preserves field cause" `Quick + (test_external_http_contract `User_keys ~success:false + ~expected_error:"E2EE user keys response: field encrypted-private-key is missing or has an invalid type" + {|{"public-key":"PRIVATE_KEY","encrypted-private-key":false}|}); + Alcotest.test_case "catalog decoder preserves nested field cause" `Quick + (test_external_http_contract `Catalog ~success:false + ~expected_error:"graph catalog: field graph-name is missing or has an invalid type" + {|{"graphs":[{"graph-id":"11111111-1111-4111-8111-111111111111","graph-name":false,"graph-ready-for-use?":true}]}|})] +;; diff --git a/swift/JournalChrome.swift b/swift/JournalChrome.swift index ccfcb60d..e163ac1f 100644 --- a/swift/JournalChrome.swift +++ b/swift/JournalChrome.swift @@ -991,7 +991,9 @@ struct JournalSheetDismissButton: SwiftUI.View { child(3).font(.footnote).frame(maxWidth: .infinity) } } + #if os(iOS) .listSectionSpacing(16) + #endif .scrollDismissesKeyboard(.interactively) #if os(iOS) .toolbar(.hidden, for: .navigationBar) diff --git a/swift/JournalCognitoNative.swift b/swift/JournalCognitoNative.swift index ca22051b..ad69ba0a 100644 --- a/swift/JournalCognitoNative.swift +++ b/swift/JournalCognitoNative.swift @@ -47,14 +47,14 @@ private final class JournalCognitoKeychain: Sendable { throw JournalCognitoFailure(category: .storage, detail: .init(source: .keychainLoad, reason: .responseTooLarge)) } do { return try JSONDecoder().decode(JournalCognitoTokens.self, from: data) } - catch { throw JournalCognitoFailure(category: .storage, detail: .init(source: .keychainLoad, reason: .invalidRecord)) } + catch { throw JournalCognitoFailure(category: .storage, detail: .init(source: .keychainLoad, reason: .invalidRecord, causes: JournalAccountDiagnostic.Cause.chain(error))) } } } func save(_ tokens: JournalCognitoTokens) async throws { try await perform { let data: Data do { data = try JSONEncoder().encode(tokens) } - catch { throw JournalCognitoFailure(category: .storage, detail: .init(source: .keychainSave, reason: .invalidRecord)) } + catch { throw JournalCognitoFailure(category: .storage, detail: .init(source: .keychainSave, reason: .invalidRecord, causes: JournalAccountDiagnostic.Cause.chain(error))) } guard data.count <= 131072 else { throw JournalCognitoFailure(category: .storage, detail: .init(source: .keychainSave, reason: .responseTooLarge)) } @@ -115,7 +115,8 @@ private final class JournalCognitoKeychain: Sendable { self?.finish(id, .failure(CancellationError())) } else { self?.finish(id, .failure(JournalCognitoFailure(category: .unavailable, - detail: .init(source: .browser, reason: .browserFailed)))) + detail: .init(source: .browser, reason: .browserFailed, + causes: error.map { JournalAccountDiagnostic.Cause.chain($0) } ?? [])))) } } } diff --git a/swift/JournalCognitoOAuth.swift b/swift/JournalCognitoOAuth.swift index c2c9fab6..e6d03e39 100644 --- a/swift/JournalCognitoOAuth.swift +++ b/swift/JournalCognitoOAuth.swift @@ -22,10 +22,10 @@ enum JournalCognitoOAuth { static func data64(_ value: String) throws -> Data { guard !value.isEmpty, value.utf8.allSatisfy({ (65...90).contains($0) || (97...122).contains($0) || (48...57).contains($0) || $0 == 45 || $0 == 95 - }) else { throw JournalCognitoError.invalidToken } + }) else { throw JournalCognitoTokenValidator.failure(.base64Invalid) } let base = value.replacingOccurrences(of: "-", with: "+").replacingOccurrences(of: "_", with: "/") guard let data = Data(base64Encoded: base + String(repeating: "=", count: (4 - base.count % 4) % 4)) - else { throw JournalCognitoError.invalidToken } + else { throw JournalCognitoTokenValidator.failure(.base64Invalid) } return data } static func random() throws -> String { @@ -97,13 +97,16 @@ struct JournalCognitoTokenResponse: Decodable { /// Verify both signatures against keys from the fixed user-pool issuer, never a JWT URL. struct JournalCognitoTokenValidator { + static func failure(_ reason: JournalAccountDiagnostic.Reason) -> JournalCognitoFailure { + JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: reason)) + } struct Key: Decodable { let kty: String; let kid: String; let alg: String?; let use: String?; let n: String; let e: String } struct Keys: Decodable { let keys: [Key] } let keys: Keys init(jwks: Data) throws { - guard jwks.count <= 262144 else { throw JournalCognitoError.invalidToken } + guard jwks.count <= 262144 else { throw Self.failure(.responseTooLarge) } keys = try JSONDecoder().decode(Keys.self, from: jwks) } private struct Header: Decodable { @@ -113,7 +116,7 @@ struct JournalCognitoTokenValidator { init(from decoder: Decoder) throws { // Reject the presence of an unsupported critical extension, including null. guard !(try decoder.container(keyedBy: Keys.self)).contains(.crit) else { - throw JournalCognitoError.invalidToken + throw JournalCognitoTokenValidator.failure(.criticalHeaderUnsupported) } fields = try Fields(from: decoder) } @@ -124,17 +127,26 @@ struct JournalCognitoTokenValidator { } private func claims(_ token: String) throws -> Claims { - guard token.utf8.count <= 32768 else { throw JournalCognitoError.invalidToken } + guard token.utf8.count <= 32768 else { throw Self.failure(.tokenTooLarge) } let parts = token.split(separator: ".", omittingEmptySubsequences: false).map(String.init) - guard parts.count == 3, - let header = try? JSONDecoder().decode(Header.self, from: JournalCognitoOAuth.data64(parts[0])), - header.fields.alg == "RS256", - keys.keys.filter({ $0.kid == header.fields.kid }).count == 1, - let key = keys.keys.first(where: { $0.kid == header.fields.kid }), key.kty == "RSA", - key.alg == nil || key.alg == "RS256", key.use == nil || key.use == "sig" - else { throw JournalCognitoError.invalidToken } + guard parts.count == 3 else { throw Self.failure(.tokenSegmentsInvalid) } + let header: Header + do { header = try JSONDecoder().decode(Header.self, from: JournalCognitoOAuth.data64(parts[0])) } + catch { + if let failure = error as? JournalCognitoFailure { throw failure } + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenHeader, reason: .responseShape, + causes: JournalAccountDiagnostic.Cause.chain(error))) + } + guard header.fields.alg == "RS256" else { throw Self.failure(.algorithmUnsupported) } + let matches = keys.keys.filter { $0.kid == header.fields.kid } + guard !matches.isEmpty else { throw Self.failure(.signingKeyMissing) } + guard matches.count == 1 else { throw Self.failure(.signingKeyAmbiguous) } + let key = matches[0] + guard key.kty == "RSA" else { throw Self.failure(.signingKeyTypeInvalid) } + guard key.alg == nil || key.alg == "RS256" else { throw Self.failure(.signingKeyAlgorithmInvalid) } + guard key.use == nil || key.use == "sig" else { throw Self.failure(.signingKeyUseInvalid) } let n = try JournalCognitoOAuth.data64(key.n), e = try JournalCognitoOAuth.data64(key.e) - guard (256...1024).contains(n.count), (1...8).contains(e.count) else { throw JournalCognitoError.invalidToken } + guard (256...1024).contains(n.count), (1...8).contains(e.count) else { throw Self.failure(.signingKeySizeInvalid) } func tlv(_ tag: UInt8, _ data: Data) -> Data { let count = data.count var length = Data() @@ -150,34 +162,61 @@ struct JournalCognitoTokenValidator { tlv(2, (data.first! >= 128 ? Data([0]) : Data()) + data) } let der = tlv(48, integer(n) + integer(e)) + var importError: Unmanaged? guard let publicKey = SecKeyCreateWithData(der as CFData, - [kSecAttrKeyType: kSecAttrKeyTypeRSA, kSecAttrKeyClass: kSecAttrKeyClassPublic] as CFDictionary, nil), - SecKeyVerifySignature(publicKey, .rsaSignatureMessagePKCS1v15SHA256, + [kSecAttrKeyType: kSecAttrKeyTypeRSA, kSecAttrKeyClass: kSecAttrKeyClassPublic] as CFDictionary, &importError) + else { + throw JournalCognitoFailure(category: .invalidToken, + detail: .init(source: .tokenValidation, reason: .signingKeyImportFailed, + causes: importError.map { JournalAccountDiagnostic.Cause.chain($0.takeRetainedValue() as Error) } ?? [])) + } + var signatureError: Unmanaged? + guard SecKeyVerifySignature(publicKey, .rsaSignatureMessagePKCS1v15SHA256, Data((parts[0] + "." + parts[1]).utf8) as CFData, - try JournalCognitoOAuth.data64(parts[2]) as CFData, nil), - let result = try? JSONDecoder().decode(Claims.self, from: JournalCognitoOAuth.data64(parts[1])) - else { throw JournalCognitoError.invalidToken } - return result + try JournalCognitoOAuth.data64(parts[2]) as CFData, &signatureError) + else { + throw JournalCognitoFailure(category: .invalidToken, + detail: .init(source: .tokenValidation, reason: .signatureInvalid, + causes: signatureError.map { JournalAccountDiagnostic.Cause.chain($0.takeRetainedValue() as Error) } ?? [])) + } + do { return try JSONDecoder().decode(Claims.self, from: JournalCognitoOAuth.data64(parts[1])) } + catch { + if let failure = error as? JournalCognitoFailure { throw failure } + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenClaims, reason: .responseShape, + causes: JournalAccountDiagnostic.Cause.chain(error))) + } } func validate(_ response: JournalCognitoTokenResponse, configuration: JournalCognitoConfiguration, nonce: String?, previous: JournalCognitoTokens?, now: Date) throws -> JournalCognitoTokens { let id = try claims(response.id_token), access = try claims(response.access_token) func expiry(_ claims: Claims, use: String) throws -> Date { - guard claims.iss == configuration.issuer, claims.token_use == use, - claims.exp.isFinite, claims.exp > now.timeIntervalSince1970, - claims.iat.isFinite, claims.iat <= now.timeIntervalSince1970 + 60 - else { throw JournalCognitoError.invalidToken } + guard claims.iss == configuration.issuer else { + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .issuerMismatch)) + } + guard claims.exp.isFinite, claims.exp > now.timeIntervalSince1970 else { + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .tokenExpired)) + } + guard claims.token_use == use else { throw Self.failure(.tokenUseMismatch) } + guard claims.iat.isFinite, claims.iat <= now.timeIntervalSince1970 + 60 + else { throw Self.failure(.issuedAtInvalid) } return Date(timeIntervalSince1970: claims.exp) } let expiryID = try expiry(id, use: "id"), expiryAccess = try expiry(access, use: "access") - guard id.aud == configuration.clientID, - access.client_id == configuration.clientID, - !id.sub.isEmpty, id.sub.utf8.count <= 512, - access.sub == id.sub, previous == nil || previous?.userID == id.sub, - nonce == nil || id.nonce == nonce, - response.token_type.lowercased() == "bearer", response.expires_in > 0, - let refresh = response.refresh_token ?? previous?.refreshToken, !refresh.isEmpty, - refresh.utf8.count <= 32768 else { throw JournalCognitoError.invalidToken } + guard id.aud == configuration.clientID, access.client_id == configuration.clientID else { + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .audienceMismatch)) + } + guard nonce == nil || id.nonce == nonce else { + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .nonceMismatch)) + } + guard let refresh = response.refresh_token ?? previous?.refreshToken, !refresh.isEmpty else { + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .refreshTokenMissing)) + } + guard !id.sub.isEmpty, id.sub.utf8.count <= 512 else { throw Self.failure(.subjectInvalid) } + guard access.sub == id.sub else { throw Self.failure(.subjectMismatch) } + guard previous == nil || previous?.userID == id.sub else { throw Self.failure(.accountMismatch) } + guard response.token_type.lowercased() == "bearer" else { throw Self.failure(.tokenTypeInvalid) } + guard response.expires_in > 0 else { throw Self.failure(.expiresInInvalid) } + guard refresh.utf8.count <= 32768 else { throw Self.failure(.refreshTokenTooLarge) } return JournalCognitoTokens(idToken: response.id_token, accessToken: response.access_token, refreshToken: refresh, userID: id.sub, issuer: configuration.issuer, clientID: configuration.clientID, expiresAt: min(expiryID, expiryAccess, now.addingTimeInterval(TimeInterval(response.expires_in)))) diff --git a/swift/JournalCognitoSession.swift b/swift/JournalCognitoSession.swift index ba7ab7a0..2a260212 100644 --- a/swift/JournalCognitoSession.swift +++ b/swift/JournalCognitoSession.swift @@ -45,9 +45,15 @@ import OSLog let value = try await storage.load() try check(version, network: network) if let value { - guard value.issuer == configuration.issuer, value.clientID == configuration.clientID, - !value.userID.isEmpty, !value.idToken.isEmpty, !value.accessToken.isEmpty, !value.refreshToken.isEmpty - else { throw JournalCognitoError.invalidToken } + func invalid(_ reason: JournalAccountDiagnostic.Reason) -> JournalCognitoFailure { + JournalCognitoFailure(category: .invalidToken, detail: .init(source: .sessionRestore, reason: reason)) + } + guard value.issuer == configuration.issuer else { throw invalid(.issuerMismatch) } + guard value.clientID == configuration.clientID else { throw invalid(.audienceMismatch) } + guard !value.userID.isEmpty else { throw invalid(.subjectInvalid) } + guard !value.idToken.isEmpty else { throw invalid(.idTokenMissing) } + guard !value.accessToken.isEmpty else { throw invalid(.accessTokenMissing) } + guard !value.refreshToken.isEmpty else { throw invalid(.refreshTokenMissing) } } // A concurrent restore may have completed a refresh or sign-in already. if !loaded { tokens = value; loaded = true } @@ -212,7 +218,8 @@ import OSLog if JournalCognitoFailure.category(of: error) != nil { throw error } let prior = JournalAccountIssue.classify(error, stage: .identityLookup) throw JournalCognitoFailure(category: .request, detail: .init(source: source, - reason: .transport, urlCode: prior.nativeDetail?.urlCode, osStatus: prior.nativeDetail?.osStatus), + reason: .transport, urlCode: prior.nativeDetail?.urlCode, osStatus: prior.nativeDetail?.osStatus, + causes: JournalAccountDiagnostic.Cause.chain(error)), accountKind: prior.kind) } } @@ -242,7 +249,8 @@ import OSLog } do { return try JSONDecoder().decode(JournalCognitoTokenResponse.self, from: data) } catch { - throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .token, reason: .responseShape, httpStatus: response.statusCode)) + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .token, reason: .responseShape, httpStatus: response.statusCode, + causes: JournalAccountDiagnostic.Cause.chain(error))) } } private func validate(_ response: JournalCognitoTokenResponse, nonce: String?, previous: JournalCognitoTokens?) async throws -> JournalCognitoTokens { @@ -261,13 +269,16 @@ import OSLog let validator: JournalCognitoTokenValidator do { validator = try JournalCognitoTokenValidator(jwks: data) } catch { - throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .jwks, reason: .responseShape, httpStatus: http.statusCode)) + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .jwks, reason: .responseShape, httpStatus: http.statusCode, + causes: JournalAccountDiagnostic.Cause.chain(error))) } do { return try validator.validate(response, configuration: configuration, nonce: nonce, previous: previous, now: now()) } catch { - throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .verificationFailed)) + if let failure = error as? JournalCognitoFailure { throw failure } + throw JournalCognitoFailure(category: .invalidToken, detail: .init(source: .tokenValidation, reason: .verificationFailed, + causes: JournalAccountDiagnostic.Cause.chain(error))) } } } diff --git a/swift/JournalLocalAccountBindingStore.swift b/swift/JournalLocalAccountBindingStore.swift index e147af6a..0bc87ea1 100644 --- a/swift/JournalLocalAccountBindingStore.swift +++ b/swift/JournalLocalAccountBindingStore.swift @@ -39,7 +39,8 @@ enum JournalLocalAccountBindingStore { } let version: Int do { version = try JSONDecoder().decode(Format.self, from: data).version } - catch { throw JournalAccountIssue(kind: .corruptBinding, stage: .bindingRead) } + catch { throw JournalAccountIssue(kind: .corruptBinding, stage: .bindingRead, + nativeDetail: .init(source: .bindingLoad, reason: .invalidRecord, causes: JournalAccountDiagnostic.Cause.chain(error))) } guard version == 2 else { throw JournalAccountIssue(kind: .unsupportedFormat, stage: .bindingRead, formatVersion: version) } @@ -48,7 +49,8 @@ enum JournalLocalAccountBindingStore { binding = try JSONDecoder().decode(Binding.self, from: data) _ = try encode(userID: binding.userId, managedSyncOrigin: binding.managedSyncOrigin) } catch { - throw JournalAccountIssue(kind: .corruptBinding, stage: .bindingRead, formatVersion: version) + throw JournalAccountIssue(kind: .corruptBinding, stage: .bindingRead, formatVersion: version, + nativeDetail: .init(source: .bindingLoad, reason: .invalidRecord, causes: JournalAccountDiagnostic.Cause.chain(error))) } // The key-management bridge accepts domain arguments, separate from the // Keychain file's generated Codable representation. diff --git a/swift/JournalNativeServices.swift b/swift/JournalNativeServices.swift index 4a74425b..6ef9f19b 100644 --- a/swift/JournalNativeServices.swift +++ b/swift/JournalNativeServices.swift @@ -121,7 +121,7 @@ private final class JournalNativeAccountStorage: Sendable { recovery = try JSONDecoder().decode(Recovery.self, from: bytes) } catch { throw JournalAccountIssue(kind: .unavailable, stage: .bindingWrite, - nativeDetail: .init(source: .recovery, reason: .invalidRecord)) + nativeDetail: .init(source: .recovery, reason: .invalidRecord, causes: JournalAccountDiagnostic.Cause.chain(error))) } guard recovery.version == 1 else { throw JournalAccountIssue(kind: .unavailable, stage: .bindingWrite, formatVersion: recovery.version, diff --git a/swift/JournalPlatformServices.swift b/swift/JournalPlatformServices.swift index 3e934ed1..ccc94f8c 100644 --- a/swift/JournalPlatformServices.swift +++ b/swift/JournalPlatformServices.swift @@ -20,32 +20,78 @@ enum JournalCognitoError: String, Error, Equatable, Sendable, LocalizedError { struct JournalAccountDiagnostic: Equatable, Sendable { enum Source: String { case keychainLoad, keychainSave, keychainAdd, keychainClear, legacyClear, browser - case token, jwks, tokenValidation, revoke, network, secureStorage, recovery + case token, jwks, tokenValidation, tokenHeader, tokenClaims, sessionRestore, revoke, network, secureStorage, recovery case bindingLoad, bindingWrite, bindingAdd, bindingRestore, bindingClear - case recoveryLoad, recoveryWrite, recoveryAdd, recoveryClear + case recoveryLoad, recoveryWrite, recoveryAdd, recoveryClear, native } enum Reason: String { case status, wrongType, responseTooLarge, invalidRecord, operationInProgress case missingAnchor, startRejected, browserFailed, redirect, httpStatus, responseShape case transport, verificationFailed, unsupportedFormat + case nonceMismatch, audienceMismatch, issuerMismatch, signatureInvalid, tokenExpired, refreshTokenMissing + case base64Invalid, tokenTooLarge, tokenSegmentsInvalid, algorithmUnsupported, criticalHeaderUnsupported + case signingKeyMissing, signingKeyAmbiguous, signingKeyTypeInvalid, signingKeyAlgorithmInvalid + case signingKeyUseInvalid, signingKeySizeInvalid, signingKeyImportFailed + case tokenUseMismatch, issuedAtInvalid, subjectInvalid, subjectMismatch, accountMismatch + case tokenTypeInvalid, expiresInInvalid, refreshTokenTooLarge, idTokenMissing, accessTokenMissing } enum OAuth: String { case invalid_request, invalid_client, invalid_grant, unauthorized_client case unsupported_grant_type, invalid_scope, access_denied, server_error, temporarily_unavailable, unknown init(safeCode: String) { self = Self(rawValue: safeCode) ?? .unknown } } + /// Bounded causal chain. Do not retain descriptions, coding keys, userInfo, + /// arbitrary domains, URLs or payloads, even for errors from unknown providers. + struct Cause: Equatable, Sendable { + enum Kind: String, Sendable { + case decodeTypeMismatch, decodeMissingKey, decodeMissingValue, decodeCorrupted + case encodeInvalidValue, cocoa, posix, url, osStatus, webAuthentication, other + } + let kind: Kind + var code: Int? = nil + var diagnostic: String { kind.rawValue + (code.map { " · code \($0)" } ?? "") } + static func chain(_ error: Error) -> [Self] { + switch error { + case DecodingError.typeMismatch: return [.init(kind: .decodeTypeMismatch)] + case DecodingError.keyNotFound: return [.init(kind: .decodeMissingKey)] + case DecodingError.valueNotFound: return [.init(kind: .decodeMissingValue)] + case DecodingError.dataCorrupted: return [.init(kind: .decodeCorrupted)] + case EncodingError.invalidValue: return [.init(kind: .encodeInvalidValue)] + default: break + } + var result: [Self] = [] + var current: NSError? = error as NSError + // The limit also bounds malformed/cyclic NSUnderlyingErrorKey chains. + while let native = current, result.count < 4 { + let kind: Kind + switch native.domain { + case NSCocoaErrorDomain: kind = .cocoa + case NSPOSIXErrorDomain: kind = .posix + case NSURLErrorDomain: kind = .url + case NSOSStatusErrorDomain: kind = .osStatus + case "com.apple.AuthenticationServices.WebAuthenticationSession": kind = .webAuthentication + default: kind = .other + } + result.append(.init(kind: kind, code: native.code)) + current = native.userInfo[NSUnderlyingErrorKey] as? NSError + } + return result + } + } let source: Source let reason: Reason var httpStatus: Int? = nil var urlCode: Int? = nil var osStatus: Int? = nil var oauth: OAuth? = nil + var causes: [Cause] = [] var diagnostic: String { "\(source.rawValue).\(reason.rawValue)" + (httpStatus.map { " · http \($0)" } ?? "") + (urlCode.map { " · url \($0)" } ?? "") + (osStatus.map { " · osstatus \($0)" } ?? "") + (oauth.map { " · oauth.\($0.rawValue)" } ?? "") + + causes.map { " · cause.\($0.diagnostic)" }.joined() } } @@ -112,7 +158,8 @@ struct JournalAccountIssue: Error, Equatable, Sendable, LocalizedError { return Self(kind: .secureStorageUnavailable, stage: stage, nativeDetail: .init(source: .secureStorage, reason: .status, osStatus: native.code)) } - return Self(kind: .unavailable, stage: stage) + return Self(kind: .unavailable, stage: stage, + nativeDetail: .init(source: .native, reason: .status, causes: JournalAccountDiagnostic.Cause.chain(error))) } } diff --git a/swift/JournalRuntime.swift b/swift/JournalRuntime.swift index a079249d..6d7e2be6 100644 --- a/swift/JournalRuntime.swift +++ b/swift/JournalRuntime.swift @@ -138,18 +138,32 @@ private let platformRequest: PlatformRequestCallback = { data, length in "Journal could not complete a native operation (\(rawValue))." } } + /// Finite startup cause codes supplied by Journal_bridge. Never native text. + enum StartupCause: Int, Sendable { + case unknown = 0, startupPayloadTruncated, startupPayloadTooLarge, startupMagicInvalid + case startupLengthInvalid, startupLengthMismatch, startupJSONInvalid + case supportDirectoryInvalid, responseBudgetInvalid, pageSizeInvalid, syncOriginInvalid + case configurationInvalid, exceptionFailure, invalidArgument, systemError + case permissionDenied, missingFile, noSpace, busy, ioError, unixError + case outOfMemory, stackOverflow, endOfFile, matchFailure, assertionFailure, notFound + case mountRejected, workerStopped, workerTerminal, workerAlreadyAttached, workerDidNotStart + var diagnostic: String { String(describing: self) } + } struct FailureDetail: Error, LocalizedError { let operation: Failure let reason: JournalSafeDiagnostic + var cause: StartupCause? = nil var errorDescription:String? { - "Journal could not complete a native operation (\(operation.rawValue) · \(reason.rawValue)). \(reason.errorDescription!)" + "Journal could not complete a native operation (\(operation.rawValue) · \(reason.rawValue)\(cause.map { " · cause.\($0.diagnostic)" } ?? "")). \(reason.errorDescription!)" } } private(set) var failureReason: JournalSafeDiagnostic? + private(set) var failureCause: StartupCause? var failureDescription:String? { - failure.map { FailureDetail(operation:$0,reason:failureReason ?? .unknown).errorDescription! } + failure.map { FailureDetail(operation:$0,reason:failureReason ?? .unknown,cause:failureCause).errorDescription! } } static func safeReason(_ error:Error) -> JournalSafeDiagnostic { + if case JournalPlatformWire.Failure.invalidPacket = error { return .invalidPayload } if case let LUIBackendError.invalidBatch(source) = error { return .backendBatch(source) } return .classify(error) } @@ -159,22 +173,30 @@ private let platformRequest: PlatformRequestCallback = { data, length in return "Journal could not finish (\(operation) · \(reason.rawValue)). \(reason.errorDescription!)" } private func record(_ operation:Failure, _ reason:JournalSafeDiagnostic) { - failure = operation; failureReason = reason + failure = operation; failureReason = reason; failureCause = nil } - private func bridgeReason(_ operation:Int32) -> JournalSafeDiagnostic { - switch journalOCamlFailureReason(operation) { - case 1: .missingHandler - case 2: .callbackFailed - case 3: .allocationFailed - case 4: .invalidInput - case 5: .startupDecodeFailed - case 6: .startupWorkerFailed - case 7: .startupMountFailed - default: .unknown + private func recordBridge(_ failure: Failure, _ operation: Int32) { + let code = journalOCamlFailureReason(operation) + let reason: JournalSafeDiagnostic + switch code { + case 1: reason = .missingHandler + case 2: reason = .callbackFailed + case 3: reason = .allocationFailed + case 4: reason = .invalidInput + case 5, 500...531: reason = .startupDecodeFailed + case 6, 600...631: reason = .startupWorkerFailed + case 7: reason = .startupMountFailed + case 700...731: reason = .startupInitializeFailed + default: reason = .unknown + } + record(failure, reason) + if (500...531).contains(code) || (600...631).contains(code) || (700...731).contains(code) { + failureCause = StartupCause(rawValue: Int(code % 100)) } } + private var reportedFailure:FailureDetail? { - failure.map { FailureDetail(operation:$0,reason:failureReason ?? .unknown) } + failure.map { FailureDetail(operation:$0,reason:failureReason ?? .unknown,cause:failureCause) } } let backend: LUIAppleBackend let platform: JournalApplicationPlatform @@ -229,7 +251,7 @@ private let platformRequest: PlatformRequestCallback = { data, length in } guard generation == identity else { throw CancellationError() } guard accepted == 1 else { - record(.start,bridgeReason(1)) + recordBridge(.start,1) let reported = reportedFailure! _ = luiOCamlStop() activeRuntime = nil @@ -289,7 +311,7 @@ private let platformRequest: PlatformRequestCallback = { data, length in started = false activeRuntime = nil stopPending = luiOCamlStop() != 1 - if stopPending { record(.stop,bridgeReason(2)) } + if stopPending { recordBridge(.stop,2) } return !stopPending } @@ -309,7 +331,7 @@ private let platformRequest: PlatformRequestCallback = { data, length in func pump() { guard started else { return } let identity = generation, accepted = journalOCamlPump() - if generation == identity, started, accepted != 1 { record(.pump,bridgeReason(3)) } + if generation == identity, started, accepted != 1 { recordBridge(.pump,3) } } /// Marshals one LJP2 request onto the platform actor and ships its response @@ -334,7 +356,7 @@ private let platformRequest: PlatformRequestCallback = { data, length in let accepted = response.withUnsafeBytes { buffer in journalOCamlPlatformFailure(buffer.baseAddress?.assumingMemoryBound(to:CChar.self),Int32(buffer.count)) } - if generation == identity, started, accepted != 1 { record(.platformFailure,bridgeReason(6)) } + if generation == identity, started, accepted != 1 { recordBridge(.platformFailure,6) } return } let accepted = response.withUnsafeBytes { buffer in @@ -342,7 +364,7 @@ private let platformRequest: PlatformRequestCallback = { data, length in buffer.baseAddress?.assumingMemoryBound(to: CChar.self), Int32(buffer.count)) } - if generation == identity, started, accepted != 1 { record(.platformResponse,bridgeReason(5)) } + if generation == identity, started, accepted != 1 { recordBridge(.platformResponse,5) } } /// Pushes one host-originated LJP2 event envelope to OCaml. @@ -354,12 +376,12 @@ private let platformRequest: PlatformRequestCallback = { data, length in buffer.baseAddress?.assumingMemoryBound(to: CChar.self), Int32(buffer.count)) } - if generation == identity, started, accepted != 1 { record(.platformEvent,bridgeReason(4)) } + if generation == identity, started, accepted != 1 { recordBridge(.platformEvent,4) } } private func handle(_ event: LUIEvent) { guard started else { return } let identity = generation, accepted = JournalLUIEvents.dispatch(event) - if generation == identity, started, accepted != 1 { record(.event,bridgeReason(7)) } + if generation == identity, started, accepted != 1 { recordBridge(.event,7) } } } diff --git a/swift/JournalRuntimeHost.swift b/swift/JournalRuntimeHost.swift index 90e2aca9..c57e7eaa 100644 --- a/swift/JournalRuntimeHost.swift +++ b/swift/JournalRuntimeHost.swift @@ -264,6 +264,9 @@ struct JournalHost: View { } private func lifecycle(backgrounded: Bool) { do { try platform.setBackgrounded(backgrounded) } - catch { Logger(subsystem: "com.logseq.journal", category: "lifecycle").error("Unable to enqueue lifecycle event") } + catch { + let diagnostic = JournalRuntime.safeDescription(error, operation: "lifecycle.enqueue") + Logger(subsystem: "com.logseq.journal", category: "lifecycle").error("\(diagnostic, privacy: .public)") + } } } diff --git a/swift/JournalSafeDiagnostic.swift b/swift/JournalSafeDiagnostic.swift index c4f170b1..3112d05f 100644 --- a/swift/JournalSafeDiagnostic.swift +++ b/swift/JournalSafeDiagnostic.swift @@ -4,7 +4,7 @@ import Foundation enum JournalSafeDiagnostic: String, Error, LocalizedError, Sendable { case unknown, missingPayload, typeMismatch, keyNotFound, valueNotFound, dataCorrupted, invalidPayload case missingHandler, callbackFailed, allocationFailed, invalidInput, missingRoot - case startupDecodeFailed, startupWorkerFailed, startupMountFailed + case startupDecodeFailed, startupWorkerFailed, startupMountFailed, startupInitializeFailed case invalidStartupConfiguration case backendInvalidBatch, backendUnsupportedProperty, backendUnknownNode, backendUnknownExtension case backendFingerprintMismatch, backendInvalidHierarchy, backendInvalidEvent, backendRegistryInvalid @@ -25,6 +25,7 @@ enum JournalSafeDiagnostic: String, Error, LocalizedError, Sendable { case .missingRoot: "The native runtime did not provide a root view. Reopen Journal." case .startupDecodeFailed: "The native startup configuration could not be decoded. Reopen Journal." case .startupWorkerFailed: "The native graph worker could not start." + case .startupInitializeFailed: "The native application could not initialize." case .startupMountFailed: "The native root view could not be mounted." case .invalidStartupConfiguration: "The native startup configuration is invalid. Reopen Journal." case .backendInvalidBatch: "The native backend rejected the patch or event batch." diff --git a/test/application_view_test.ml b/test/application_view_test.ml index 72a14af0..64c0d545 100644 --- a/test/application_view_test.ml +++ b/test/application_view_test.ml @@ -2783,8 +2783,11 @@ let run_favorites_native_visibility | Some (`String payload) -> let json = Yojson.Safe.from_string payload in Yojson.Safe.Util.member "completion" json = `String operation - && Yojson.Safe.Util.member "error" json = - `String "Import was interrupted. Select the file again." + && (match Yojson.Safe.Util.member "error" json with + | `String message -> + String.starts_with ~prefix:"Import: Fixture interrupted import" message + && String.ends_with ~suffix:"Import outcome is unknown. Select the file again." message + | _ -> false) | _ -> false) props false); Alcotest.(check bool) "non-staged import cannot offer retained-byte retry" true (Option.is_none (find "accessibility-label" "Retry imports")))); @@ -6486,7 +6489,7 @@ let test_failed_init_disposes_context () = |> Result.get_ok |> Journal_startup.encode |> Result.get_ok |> Bytes.to_string in Fun.protect ~finally:(fun () -> Atomic.set release true; ignore(hooks.dispose()); Option.iter Logseq_db_worker_lui.Journal_worker_runtime.stop !client) (fun () -> - let failed = try ignore(hooks.init 2 2 startup); false with Failure _ -> true in + let failed = try ignore(hooks.init 2 2 startup); false with _ -> true in Alcotest.(check bool) "init exception reaches native boundary" true failed; Alcotest.(check int) "failed startup releases its mounted context" 0 (hooks.root_node ())) ;; diff --git a/test/journal_graph_runtime_locality_test.ml b/test/journal_graph_runtime_locality_test.ml index 9926d57d..ae57fc6f 100644 --- a/test/journal_graph_runtime_locality_test.ml +++ b/test/journal_graph_runtime_locality_test.ml @@ -82,7 +82,7 @@ module Worker_producer = struct | Ok(W.Answered(Error W.Response_budget_exceeded))->Production_runtime.receive_worker_read runtime ~request_id:request (Production_runtime.Read_failed{code=Logseq_db_worker.Error.Response_too_large;message="oversized fixture"}) | Ok(W.Answered(Error(W.Db_error error)))->Production_runtime.receive_worker_read runtime ~request_id:request - (Production_runtime.Read_failed{code=Logseq_db_worker.Error.code error;message=Logseq_db_worker.Error.message error}) + (Production_runtime.Read_error error) | _->Production_runtime.fail_request runtime request ~message:"Typed fixture response rejected") |Some(Production_runtime.Acknowledge{batch})-> let operation=W.Acknowledge_changes batch in @@ -5628,6 +5628,88 @@ let test_favorite_epoch_preserves_new_queue () = (* Worker admission belongs to transport. Test only that missing boundary, with a real Runtime output; do not duplicate DB/effect-runner coverage. *) +(* Transport owns admission refusal, and Runtime owns preservation of the + correctly typed DB completion. Neither is produced by Root_navigation. *) +let test_timeline_admission_keeps_refusal_cause () = + let module E=Logseq_db_worker.Error in + List.iter (fun (refusal,expected) -> + let runtime=Production_runtime.create () in + set_calendar runtime; + let output=Production_runtime.submit runtime (Journal_graph_request.Load_feed + {cursor=None;day_limit=7;root_limit=32;request_generation=1L}) in + let delivery=Journal_graph_transport.deliver ~runtime ~send:(fun _->refusal) output in + match delivery.responses with + | [{Production_runtime.payload=Feed_failed {failure=Worker_failure failure;_}}] -> + Alcotest.(check string) "date read operation" "listTimelineDates" failure.operation; + Alcotest.(check (option string)) "precise admission cause" (Some expected) + (E.trace failure.error).origin.code; + if refusal=Journal_graph_transport.Full then + Alcotest.(check bool) "queue full is not database closed" false + (E.code failure.error=E.Closed_session) + | _ -> Alcotest.fail "expected original feed failure") + Journal_graph_transport.[Full,"queueFull";Not_ready,"notReady";Stopping,"stopping"] +;; + +let test_timeline_read_preserves_typed_origin () = + let module E=Logseq_db_worker.Error in + List.iter(fun category-> + let runtime=Production_runtime.create () in + set_calendar runtime; + let output=Production_runtime.submit runtime(Journal_graph_request.Load_feed + {cursor=None;day_limit=7;root_limit=32;request_generation=1L}) in + let request=only "timeline dates" output.requests in + let origin=E.create_cause ~component:E.Read_model ~operation:"snapshot" + ~code:(Some category) ~message:("Typed cause: " ^ category) |> Result.get_ok in + let original=E.create_with_origin ~code:E.Closed_session ~message:"Timeline read unavailable" + ~details:[E.{name="phase";value=Detail_string "snapshot"}] ~origin |> Result.get_ok in + let result=Production_runtime.receive_worker_read runtime ~request_id:request + (Production_runtime.Read_error original) in + match result.responses with + | [{Production_runtime.payload=Feed_failed {failure=Worker_failure failure;_}}] -> + Alcotest.(check string) "same error including trace and details" + (Yojson.Safe.to_string(E.to_yojson original)) + (Yojson.Safe.to_string(E.to_yojson failure.error)); + Alcotest.(check int) "late failure remains retired" 0 + (List.length(Production_runtime.receive_worker_read runtime ~request_id:request + (Production_runtime.Read_error original)).responses) + | _ -> Alcotest.fail "expected typed feed failure") + ["Database_closed";"Snapshot_released";"Snapshot_generation_invalidated"] +;; + +let test_reference_and_favorites_preserve_typed_origin () = + let module E=Logseq_db_worker.Error in + let module A=Application.Root_navigation in + let check runtime request = + let origin=E.create_cause ~component:E.Storage ~operation:"sqliteStep" + ~code:(Some "SQLITE_CORRUPT") ~message:"SQLite reported corrupt storage" + |> Result.get_ok in + let original=E.create_with_origin ~code:E.Corrupt_storage + ~message:"Read failed" ~details:[] ~origin |> Result.get_ok in + let output=deliver_sample runtime (sample request (failed_sample (W.Db_error original))) in + let response=List.filter(fun (response:Runtime.response)->match response.payload with + | Diagnostic _ -> true | _ -> false) output.responses + |> only "typed diagnostic event" in + (match response.payload with + | Diagnostic failure-> + Alcotest.(check string) "original cause remains intact" + (Yojson.Safe.to_string(E.to_yojson original)) + (Yojson.Safe.to_string(E.to_yojson failure.error)) + | _ -> assert false); + let state,_=A.step (A.create ~graph_generation:7) (Completed response) in + Alcotest.(check (option string)) "visible diagnostic retains storage cause" + (Some "SQLITE_CORRUPT") (List.assoc_opt "Origin code" (A.diagnostic_rows state)) + in + let reference_runtime=Runtime.create () in + let request=reference_feed reference_runtime reference |> check_target_request reference_runtime in + check reference_runtime request; + let favorites_runtime=Runtime.create () in + set_calendar favorites_runtime; + let request=Runtime.submit favorites_runtime (Load_favorites + {graph_generation=7;request_generation=1L;limit=50;cursor=None}) + |> fun output->only "favorites read" output.requests in + check favorites_runtime request +;; + let test_rejected_delivery_retires_unsent_queue () = List.iter (fun refusal -> let runtime = Runtime.create () in @@ -8431,6 +8513,9 @@ let () = Alcotest.test_case "typed insertion admission" `Quick test_child_typed_insert_admission; Alcotest.test_case "unknown Child preserves wire across reset" `Quick (fun ()->test_child_unknown_wire_survives_reset ()); Alcotest.test_case "unknown Child status preserves preconditions across reset" `Quick (fun ()->test_child_unknown_wire_survives_reset ~status:true ()); + Alcotest.test_case "reference and favorites typed origins reach Diagnostics" `Quick test_reference_and_favorites_preserve_typed_origin; + Alcotest.test_case "timeline read preserves typed origin" `Quick test_timeline_read_preserves_typed_origin; + Alcotest.test_case "timeline admission retains cause" `Quick test_timeline_admission_keeps_refusal_cause; Alcotest.test_case "typed admission refusal retains graph" `Quick test_typed_child_planning_refusal_retains_transport; Alcotest.test_case "failed physical continuations retire before delayed recovery" `Quick test_child_error_continuations_retire; Alcotest.test_case "unbound child refusal remains known not admitted" `Quick test_unbound_child_refusal_is_not_admitted]) diff --git a/test/journal_routes_test.ml b/test/journal_routes_test.ml index 479ce7e7..02998659 100644 --- a/test/journal_routes_test.ml +++ b/test/journal_routes_test.ml @@ -4183,7 +4183,12 @@ let test_staged_file_adapter_guard_and_failures () = Unix.mkdir blocked 0o700; Journal_asset_import.discard_staged_file (descriptor 0 kept); require (Sys.file_exists kept) "non-import file removed"; - Journal_asset_import.discard_staged_file (descriptor 1 blocked); + let failures=ref [] in + Journal_asset_import.discard_staged_file ~on_failure:(fun message->failures:=message::!failures) (descriptor 1 blocked); + require(List.length !failures=1) "blocked staging cleanup lost its cause"; + let diagnostic=Logseq_db_types.Failure_diagnostic.decode(List.hd !failures) |> Option.get in + require(Logseq_db_types.Failure_diagnostic.reason diagnostic="cleanupFailed") "cleanup reason lost"; + require(not(String.contains (List.hd !failures) '/')) "cleanup diagnostic exposed a path"; Journal_asset_import.discard_staged_file (descriptor 2 owned); require (not (Sys.file_exists owned)) "matched file not removed after previous failure"; Journal_asset_import.discard_staged_file (descriptor 2 owned); @@ -4655,6 +4660,51 @@ module Child_chain_fixture = struct | Ignore ->fail "prefix settlement ignored its current operation" end +(* Transport owns admission of Runtime's canonical follow-up; the pure insertion + reducer cannot turn an auxiliary diagnostic into a fatal transport outcome. *) +let test_child_canonical_admission_keeps_recovery () = + let module F=Child_chain_fixture in + let module R=Journal_graph_runtime in + let module T=Journal_graph_transport in + let module E=Logseq_db_worker.Error in + let fixture=F.make () in + let runtime=R.create ~localtime:Unix.gmtime () in + let calendar=Journal_calendar.Sampler.create + ~clock:(fun()->Int64.to_float(Journal_time.instant_unix_ms fixture.command.creation_time)/.1000.) + ~localtime:Unix.gmtime () |> Journal_calendar.Sampler.sample |> Result.get_ok in + R.set_calendar runtime calendar; + let only output=match output.R.requests with [request]->request + |_->fail "expected one canonical child lifecycle request" in + let initial=R.submit ~child_chain:fixture.initial runtime + (Journal_graph_request.Create_child fixture.command) |> only in + let intent=R.worker_intent runtime initial fixture.session |> Result.get_ok in + require(R.bind_mutation runtime initial ~presentation:fixture.presentation ~intent) + "original child mutation did not bind"; + let canonical=R.receive_mutation runtime initial (fixture.confirm 0 intent fixture.presentation) in + let canonical_id=only canonical in + require(R.worker_read runtime canonical_id<>None) "committed child did not reach its canonical read"; + let sent=ref [] in + let delivered=T.deliver ~runtime ~send:(fun id->sent:=id::!sent;T.Full) canonical in + require(!sent=[canonical_id]) "transport did not refuse exactly the canonical read"; + require(delivered.error=None) "canonical read backpressure became a fatal transport error"; + require(List.exists(fun response->match response.R.payload with + | Child_waiting _ -> true | _->false) delivered.responses) + "committed child lost its waiting/recovery response"; + require(not(List.exists(fun response->match response.R.payload with + | Child_failed _ | Mutation_failed _ -> true | _->false) delivered.responses)) + "auxiliary canonical-read diagnostic became a business rejection"; + let failure=List.find_map(fun response->match response.R.payload with + | Diagnostic failure when failure.request_id=canonical_id -> Some failure + | _->None) delivered.responses |> Option.get in + require(failure.operation="readCanonicalChildParent") + "canonical read failure lost its owning operation"; + let error=failure.error in + let origin=(E.trace error).origin in + require(E.code error=E.Storage_busy && origin.operation="requestAdmission" && origin.code=Some "queueFull") + "canonical read backpressure lost its concrete admission cause" +;; +let () = test_child_canonical_admission_keeps_recovery () + let test_capture_oversize_is_editable_before_admission () = let source=String.make (Worker_pure_data.maximum_title_bytes+1) 'x' in let original=Journal_capture.create ~session_number:5518L ~source in diff --git a/test/worker_coordination_contract_test.py b/test/worker_coordination_contract_test.py index 76be0a10..ef22db7f 100644 --- a/test/worker_coordination_contract_test.py +++ b/test/worker_coordination_contract_test.py @@ -33,7 +33,7 @@ | W.Confirmed _ -> 3 let reply = function | W.Not_admitted _ -> 0 - | W.Interrupted (W.Cancelled | W.Transport_lost | W.Worker_stopped) -> 1 + | W.Interrupted (W.Cancelled | W.Transport_lost _ | W.Worker_stopped) -> 1 | W.Answered (Error _) -> 2 | W.Answered (Ok progress) -> phase progress """, @@ -103,7 +103,7 @@ | Some (W.Answered (Ok None)) -> `Empty | Some (W.Not_admitted (W.Full | W.Not_ready | W.Stopping)) -> `Retry | Some (W.Not_admitted (W.Unsupported_request operation)) -> `Unsupported operation - | Some (W.Interrupted (W.Cancelled | W.Transport_lost | W.Worker_stopped)) -> `Interrupted + | Some (W.Interrupted (W.Cancelled | W.Transport_lost _ | W.Worker_stopped)) -> `Interrupted | Some (W.Answered (Error error)) -> `Error error let rows (slice : W.block W.slice) = List.map (fun block -> W.block_uuid block, W.block_title block) slice.items diff --git a/test/worker_coordination_pure_test.ml b/test/worker_coordination_pure_test.ml index 1458aa30..d2c42f12 100644 --- a/test/worker_coordination_pure_test.ml +++ b/test/worker_coordination_pure_test.ml @@ -363,7 +363,7 @@ let test_import_source_recovery_owner_cursor_and_ack () = let admitted=(C.step planned.next(C.Admitted(imported,Id.Worker.Request_id.of_int64 9889L))).next in check(Rrbvec.is_empty(C.pending_import_source_intents(C.view admitted))) "source lookup raced the admitted original Import"; - let unknown=consume planned.next imported(W.Interrupted W.Transport_lost) in + let unknown=consume planned.next imported(W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))) in check(C.import_source_pending(C.view unknown) ~operation:source.operation)"lost import reply did not retain its source"; let operation=W.Recover_import_inputs{after=None;maximum_items=2;maximum_bytes=65536} in let recovery,planned=C.request unknown session operation |> Result.get_ok in @@ -662,7 +662,7 @@ let test_rejected_imports_release_capacity_after_fresh_readback () = done; let original=intent 261 in let ticket=request(W.Import original) in - complete ticket(W.Interrupted W.Transport_lost);apply(C.Forget ticket); + complete ticket(W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption")));apply(C.Forget ticket); let read=request(W.Confirm_import_source original) in complete read(W.Answered(Ok None));apply(C.Forget read); check(C.import_source_pending(C.view !state) ~operation:(W.asset_import_intent_id original)) @@ -747,7 +747,7 @@ let test_new_host_source_proof_retires_only_identical_unknown () = let intent=W.asset_import_intent session source |> Result.get_ok in let ticket,planned=C.request(ready I.binding projection)session(W.Import intent) |> Result.get_ok in let old=(C.step planned.next(C.Admitted(ticket,Id.Worker.Request_id.one))).next in - let old=(C.step old(C.Completed(ticket,W.Interrupted W.Transport_lost))).next in + let old=(C.step old(C.Completed(ticket,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))))).next in let old=(C.step old(C.Forget ticket)).next in let old=(C.step old C.Shutdown_requested).next in let old=(C.step old(C.Binding_release_completed(I.binding,Ok()))).next in @@ -816,7 +816,7 @@ let test_import_proof_consumption_properties () = if action mod 2=0 && not delivered.(which) then ( delivered.(which)<-true;state:=(C.step !state(C.Import_source_reconciled proof)).next) else if action mod 2=1 && not terminal.(which) then ( - terminal.(which)<-true;state:=(C.step !state(C.Completed(ticket,W.Interrupted W.Transport_lost))).next; + terminal.(which)<-true;state:=(C.step !state(C.Completed(ticket,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))))).next; state:=(C.step !state(C.Forget ticket)).next); let consumed=C.import_source_proof_consumed proof(views()) in check(consumed=(delivered.(0)&&delivered.(1))) "proof return skipped an old Host acknowledgement"; @@ -2173,7 +2173,7 @@ let test_durable_recovery_and_ack_purity () = "known rejection ACK did not carry the pure no-commit decision"; state:=planned.next; state:=(C.step !state(C.Admitted(ack,Id.Worker.Request_id.of_int64 13L))).next; - state:=(C.step !state(C.Completed(ack,W.Interrupted W.Transport_lost))).next; + state:=(C.step !state(C.Completed(ack,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))))).next; check(C.intent_progress(C.view !state)intent<>None) "interrupted ACK erased durable ownership"; state:=(C.step !state(C.Forget ack)).next; let ack,planned=C.request !state session(W.Acknowledge_intent intent) |> Result.get_ok in @@ -2358,7 +2358,7 @@ let test_committed_ack_requires_current_confirmation () = check(C.intent_progress(C.view late)(W.Mutation_intent intent)=None) "pre-ACK mutation completion resurrected settled intent"; check(match C.reply(C.view late)retry with Some(W.Answered(Error _))->true|_->false) "pre-ACK mutation completion was published as fresh ownership"; - let late=(C.step acknowledged(C.Completed(retry,W.Interrupted W.Transport_lost))).next in + let late=(C.step acknowledged(C.Completed(retry,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))))).next in check(C.intent_progress(C.view late)(W.Mutation_intent intent)=None) "pre-ACK interruption resurrected Unknown intent"; let late=(C.step acknowledged(C.Completed(again,W.Answered(Ok confirmation)))).next in check(C.intent_progress(C.view late)(W.Mutation_intent intent)=None) "pre-ACK confirmation resurrected settled intent"; @@ -2387,13 +2387,13 @@ let test_committed_ack_requires_current_confirmation () = List.iter(fun action-> let event=match action with | 0->C.Completed(retry,W.Answered(Ok(W.Submission_committed receipt))) - | 1->C.Completed(retry,W.Interrupted W.Transport_lost) + | 1->C.Completed(retry,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))) | 2->C.Cancel retry | 3->C.Forget retry | 4->C.Completed(again,W.Answered(Ok confirmation)) - | 5->C.Completed(again,W.Interrupted W.Transport_lost) + | 5->C.Completed(again,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))) | 6->C.Cancel again | 7->C.Forget again | 8->C.Completed(old_ack,W.Answered(Ok())) - | 9->C.Completed(old_ack,W.Interrupted W.Transport_lost) + | 9->C.Completed(old_ack,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))) | 10->C.Cancel old_ack | 11->C.Forget old_ack | 12->C.Completed(independent,W.Answered(Ok(W.Submission_committed other_receipt))) | _->C.Drain in @@ -2457,7 +2457,7 @@ let test_durable_recovery_ack_properties () = state:=next.next;acks:=ticket::List.take 7 !acks) | 6->ack n(fun ticket->step(C.Admitted(ticket,Id.Worker.Request_id.of_int64(Int64.of_int(200+n))))) | 7->ack n(fun ticket->step(C.Completed(ticket,W.Answered(Ok())))) - | 8->ack n(fun ticket->step(C.Completed(ticket,W.Interrupted W.Transport_lost))) + | 8->ack n(fun ticket->step(C.Completed(ticket,W.Interrupted (W.Transport_lost (W.Service_failed "synthetic transport interruption"))))) | 9->ack n(fun ticket->step(C.Cancel ticket)) | 10->ack n(fun ticket->step(C.Forget ticket)) | _->step C.Drain)trace in @@ -2824,6 +2824,30 @@ let test_observation_notice_plans_one_refresh () = check (count late = 0) "retired interest accepted a late invalidation" ;; +let test_observation_cleanup_admission_causes () = + let issuer,raw,projection,session=fixture 7259L in + let module I=(val issuer:Worker_effect_authority.ISSUER) in + let operation=W.Observe(W.read_budget session ~maximum_items:1 ~maximum_bytes:65536 |> Result.get_ok,W.Graph_info) in + let ticket,planned=C.request(ready I.binding projection)session operation |> Result.get_ok in + let response=pure_observation_response issuer raw session ticket operation 0L "projection:cleanup-cause" in + let value=match I.reply session ~request_id:(C.ticket_id ticket)operation response |> Result.get_ok with + |W.Answered(Ok value)->value|_->failwith "cleanup cause fixture" in + let admitted=(C.step planned.next(C.Admitted(ticket,Id.Worker.Request_id.one))).next in + let completed=(C.step admitted(C.Completed(ticket,W.Answered(Ok value)))).next in + let released=C.step completed(C.Observation_returned value.observation) in + let finalized=C.step released.next(C.Observation_release_completed(session,C.ticket_id ticket,Ok())) in + List.iter(fun(admission,cause)-> + let release=C.step released.next(C.Observation_release_not_admitted(session,C.ticket_id ticket,admission)) in + let finalize=C.step finalized.next(C.Observation_finalize_not_admitted(session,C.ticket_id ticket,admission)) in + check(C.observation_error(C.view release.next)value.observation=Some(cause "Observation cleanup")) + "cleanup admission discarded its specific reason"; + check(C.observation_error(C.view finalize.next)value.observation=Some(cause "Observation finalization")) + "finalization admission discarded its specific reason") + [W.Not_ready,(fun operation->W.Service_failed(operation^": worker not ready")); + W.Stopping,(fun operation->W.Service_failed(operation^": worker stopping")); + W.Unsupported_request "synthetic unsupported cleanup",(fun operation->W.Unsupported_operation(operation^": synthetic unsupported cleanup"))] +;; + let test_observation_cleanup_failure_waits_for_explicit_retry () = let issuer, _, projection, session = fixture 725L in let module I = (val issuer : Worker_effect_authority.ISSUER) in @@ -4848,6 +4872,7 @@ let test_binding_rebind_queued_read_and_failure () = ;; let () = + if Array.mem "--error-causes-only" Sys.argv then (test_observation_cleanup_admission_causes();exit 0); if Array.mem "--unconfirmed-demand-only" Sys.argv then (test_unconfirmed_asset_registration_cleanup();exit 0); if Array.mem "--live-read-rebind-only" Sys.argv then (test_live_binding_replans_ordinary_reads(); test_live_binding_read_owner_and_cancellation_fences();test_binding_rebind_queued_read_and_failure();exit 0); if Array.mem "--retry-release-error-only" Sys.argv then (test_retry_binding_release_error_is_observable();exit 0); @@ -4892,6 +4917,7 @@ let () = test_observation_cancel_requires_ack (); test_observation_read_failure_and_refresh (); test_observation_notice_plans_one_refresh (); + test_observation_cleanup_admission_causes (); test_observation_cleanup_failure_waits_for_explicit_retry (); test_observation_properties (); test_observation_full_batch_does_not_self_wake (); diff --git a/test/worker_coordination_runtime_test.ml b/test/worker_coordination_runtime_test.ml index 7ee6b267..c2339689 100644 --- a/test/worker_coordination_runtime_test.ml +++ b/test/worker_coordination_runtime_test.ml @@ -381,6 +381,108 @@ let with_client ?epoch_ms ?favorites ?assets ?statuses ?change_max_items ?prepar (* The pure route reducer has no manager push or host handoff input. Drive public mounted hooks over the actual Service, DB and resource owners. *) +(* These failures originate in mailbox-outcome decoding, outside the pure + reducer. Exercise the public runner with a real admitted request, then replace + only its transport outcome; the reducer never receives a fabricated error. *) +let test_cleanup_transport_causes () = + let failures=ref [] in + let check condition message=if not condition then failures:=message::!failures in + let run stage outcome suffix = with_client (fun client -> + let events=Queue.create () and state=ref(C.initial ~client_id) in + let runner=R.create ~client ~managed_origin:"https://api.logseq.io" ~user_id:"user-1" + ~post:(fun event->Queue.add event events) ~publish:(fun _->()) |> Result.get_ok in + let injecting=ref false and captured=ref [] and delivery_errors=ref [] and imported=ref None in + Mailbox.on_event client(fun event-> + let event=match event,!injecting with + |Mailbox.Response response,true->Mailbox.Response{response with outcome} + |_->event in + let delivery=R.on_event runner event in + if !injecting then delivery_errors:=delivery.error::!delivery_errors); + let apply transition=state:=transition.C.next;List.iter(fun instruction-> + (match instruction,stage with + |C.Release_observation_registration _,"Observation release" + |C.Finalize_observation_registration _,"Observation finalization" + |C.Release_binding _,"Binding unregister" + |C.Send(_,_,W.Import _),"Import"->injecting:=true + |_->());R.submit runner instruction)transition.instructions in + let pump ()= + Mailbox.Private.deliver client ~max_events:64; + let count=Queue.length events in + for _=1 to count do + let event=Queue.take events in + if !injecting then captured:=event::!captured else apply(C.step !state event) + done in + let wait message predicate= + let deadline=Unix.gettimeofday()+.4. in + while not(predicate()) && Unix.gettimeofday()match C.bootstrap(C.view !state)with W.Bootstrap_not_ready _->true|_->false); + ignore(Mailbox.send client(Service.Client_command(Restore_local_account{user_id="user-1"}))); + let session ()=Option.bind(C.projection(C.view !state))(fun projection-> + List.find_map(fun resident->if resident.W.phase=W.Open then Some resident.session else None)(W.residents projection)) in + wait "cleanup fixture session" (fun()->Option.is_some(session())); + let session=Option.get(session()) in + pump(); + let push notice=R.on_event runner(Mailbox.Push{ + runtime_epoch=Mailbox.runtime_epoch client;worker_generation=Mailbox.worker_generation client; + push_sequence=Id.Worker.Push_sequence.of_int64 999L;topic=Service.asset_topic; + payload=Service.Asset_notice(Read.For_worker.session_asset_scope session,notice)}) in + require((push Service.Asset_capacity_available).error=None) + "valid asset notice became a transport error"; + check((push(Service.Asset_demand_accepted "foreign-demand-owner")).error=Some W.Revoked_session) + "asset notice decoding erased its typed ownership cause"; + if stage="Binding unregister" then apply(C.step !state C.Shutdown_requested) + else if stage="Import" then ( + let source:Logseq_db_types.Asset_import.t={ + operation=uuid "9c000000-0000-4000-8000-000000009921"; + asset=uuid "9c000000-0000-4000-8000-000000009922";target=Test_support.authoritative_block_uuid; + local_mutation=uuid "9c000000-0000-4000-8000-000000009923"; + metadata_mutation=uuid "9c000000-0000-4000-8000-000000009924"; + replace_reference=None;source_file="/synthetic-missing/import.png";title="Synthetic import";file_type="png"} in + let intent=W.asset_import_intent session source |> Result.get_ok in + imported:=Some intent; + let _,plan=C.request !state session(W.Import intent) |> Result.get_ok in apply plan) + else ( + let budget=W.read_budget session ~maximum_items:1 ~maximum_bytes:65536 |> Result.get_ok in + let ticket,plan=C.request !state session(W.Observe(budget,W.Graph_info)) |> Result.get_ok in + apply plan;wait "cleanup fixture observation" (fun()->Option.is_some(C.reply(C.view !state)ticket)); + let handle=match C.reply(C.view !state)ticket with Some(W.Answered(Ok value))->value.W.observation + |_->failwith "cleanup fixture observation failed" in + apply(C.step !state(C.Forget ticket)); + let released,plan=C.request !state session(W.Release handle) |> Result.get_ok in + apply plan; + if stage="Observation finalization" then ( + wait "cleanup fixture release" (fun()->Option.is_some(C.reply(C.view !state)released)); + apply(C.step !state(C.Forget released)); + apply(C.step !state(C.Observation_returned handle)))); + if stage="Import" then ( + wait "Import completion missing" (fun()->List.exists(function C.Completed _->true|_->false)!captured); + check(List.mem (Some(W.Service_failed("Import: "^suffix))) !delivery_errors) + "Import transport failure erased its diagnostic cause"; + check(List.exists(function + |C.Completed(_,W.Interrupted(W.Transport_lost cause))->cause=W.Service_failed("Import: "^suffix) + |_->false)!captured) "Import reply erased its typed interruption cause"; + List.iter(fun event->state:=(C.step !state event).next)(List.rev !captured); + check(C.import_progress(C.view !state)(Option.get !imported)=Some W.Import_unknown) + "Import transport failure falsely settled its unknown outcome") + else ( + let result=function + |C.Observation_release_completed(_,_,result) when stage="Observation release"->Some result + |C.Observation_finalize_completed(_,_,result) when stage="Observation finalization"->Some result + |C.Binding_release_completed(_,result) when stage="Binding unregister"->Some result + |_->None in + wait (stage^" completion missing") (fun()->List.exists(fun event->Option.is_some(result event))!captured); + check(List.exists(fun event->result event=Some(Error(W.Service_failed(stage^": "^suffix))))!captured) + (stage^" erased its backend cause"))) in + List.iter(fun stage->List.iter(fun(outcome,suffix)->run stage outcome suffix) + [Mailbox.Failed "synthetic cleanup backend refused retirement","synthetic cleanup backend refused retirement"; + Mailbox.Cancelled,"cancelled before retirement acknowledgement"; + Mailbox.Shutdown,"worker stopped before retirement acknowledgement"]) + ["Observation release";"Observation finalization";"Binding unregister"]; + run "Import" (Mailbox.Failed "synthetic import transport lost") "synthetic import transport lost"; + require(!failures=[]) (String.concat "; " (List.rev !failures)) +;; + let test_application_current_close ~replacement () = let third=uuid "9c000000-0000-4000-8000-000000000004" and fourth=uuid "9c000000-0000-4000-8000-000000000005" in @@ -4423,6 +4525,8 @@ let test_retired_query_has_correlated_response () = ;; let () = + test_cleanup_transport_causes(); + if Array.mem "--cleanup-causes-only" Sys.argv then exit 0; if Array.mem "--read-revocation-only" Sys.argv then (test_retired_query_has_correlated_response();exit 0); if Array.mem "--application-close-only" Sys.argv then (test_application_current_close ~replacement:false();test_application_current_close ~replacement:true();exit 0); test_application_current_close ~replacement:false(); diff --git a/tool/test_journal_runtime_failure.py b/tool/test_journal_runtime_failure.py index 78d4ffff..41aa950b 100644 --- a/tool/test_journal_runtime_failure.py +++ b/tool/test_journal_runtime_failure.py @@ -30,14 +30,14 @@ bridge.write_text(''' #include static int mode; -int32_t journal_ocaml_failure_reason(int32_t operation) { return 2; } +int32_t journal_ocaml_failure_reason(int32_t operation) { return mode >= 500 ? mode : 2; } static int stops; static int platform_calls; int32_t fixture_platform_count(void) { return platform_calls; } int32_t fixture_stop_count(void) { return stops; } void fixture_set_mode(int32_t value) { mode = value; } int32_t lui_ocaml_start(void (*patch)(const char *), int32_t p, int32_t h, const char *d, int32_t n) { - if (mode == 1) return 0; + if (mode == 1 || mode >= 500) return 0; patch(mode == 2 ? "invalid" : "valid"); return 1; } int32_t lui_ocaml_stop(void) { stops++; return mode == 7 ? 0 : 1; } @@ -60,7 +60,7 @@ def run(args): str(ROOT / "apple-tests/platform-services/JournalRuntimeFailureTests.swift"), str(work / "bridge.o"), "-o", str(executable)]) failures = [] - for scenario in ["success", "host-lifetime", "rejected", "invalid-patch", "pump", "platform-event", "platform-response", "platform-failure", "stopped-response", "stopped-failure", "stop-retry", "stop-failure"]: + for scenario in ["startup-truncated", "startup-magic", "worker-permission", "worker-space", "success", "host-lifetime", "rejected", "invalid-patch", "pump", "platform-event", "platform-response", "platform-failure", "stopped-response", "stopped-failure", "stop-retry", "stop-failure"]: result = subprocess.run([str(executable), scenario], cwd=work, capture_output=True, text=True) print(f"runtime {scenario}: exit={result.returncode}") if result.returncode: