From ece37bd4d6acf24b2ea5138555bbfd18d7f9f3b8 Mon Sep 17 00:00:00 2001 From: Timon Date: Sun, 15 Nov 2026 23:30:20 +0000 Subject: [PATCH 1/2] Unpack an opened file's resources straight into the resource storage --- document/container/src/archive.rs | 19 ++++++-- document/container/src/archive/xz.rs | 8 ++-- document/container/src/archive/zip.rs | 8 ++-- .../document/storage_tests/lifecycle_tests.rs | 43 +++++++++++++++++++ .../messages/portfolio/document_storage_io.rs | 15 ++++++- 5 files changed, 80 insertions(+), 13 deletions(-) diff --git a/document/container/src/archive.rs b/document/container/src/archive.rs index 951708d5b65..6df8d3864f7 100644 --- a/document/container/src/archive.rs +++ b/document/container/src/archive.rs @@ -54,11 +54,18 @@ pub trait Archive { fn writer(output: W) -> Result>; + fn unpack(source: R, sink: &mut UnpackEntry) -> Result<()>; + /// Read entries from `source` and write each into `dest`, streaming so neither the full /// archive nor the full container ever sits in memory at once. - fn open(source: R, dest: &mut C) -> Result<()>; + fn open(source: R, dest: &mut C) -> Result<()> { + Self::unpack(source, &mut |path, size, fill| dest.write_sized_non_blocking(path, size, fill)) + } } +/// See [`Archive::unpack`]. +pub type UnpackEntry<'a> = dyn FnMut(&str, usize, &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> + 'a; + pub trait ArchiveWriter: Sized { type Sink; @@ -100,12 +107,18 @@ impl ArchiveFormat { /// Errors if the bytes are neither a recognized xz nor zip archive. #[cfg(any(feature = "xz", feature = "zip"))] pub fn open_auto(bytes: &[u8], dest: &mut C) -> Result<()> { + unpack_auto(bytes, &mut |path, size, fill| dest.write_sized_non_blocking(path, size, fill)) +} + +/// Like [`open_auto`], but hands each entry to `sink` instead of writing it into a container. +#[cfg(any(feature = "xz", feature = "zip"))] +pub fn unpack_auto(bytes: &[u8], sink: &mut UnpackEntry) -> Result<()> { let source = std::io::Cursor::new(bytes); match ArchiveFormat::detect(bytes) { #[cfg(feature = "xz")] - Some(ArchiveFormat::Xz) => Xz::open(source, dest), + Some(ArchiveFormat::Xz) => Xz::unpack(source, sink), #[cfg(feature = "zip")] - Some(ArchiveFormat::Zip) => Zip::open(source, dest), + Some(ArchiveFormat::Zip) => Zip::unpack(source, sink), #[allow(unreachable_patterns)] Some(format) => Err(ContainerError::Codec(format!("tried to open {format:?}, but the binary was compiled without the feature enabled "))), None => Err(ContainerError::Codec("unrecognized archive format (not xz or zip) ".into())), diff --git a/document/container/src/archive/xz.rs b/document/container/src/archive/xz.rs index 383a045b29f..f3d5e1c1c2e 100644 --- a/document/container/src/archive/xz.rs +++ b/document/container/src/archive/xz.rs @@ -1,7 +1,7 @@ //! Xz-compressed tarball archive codec. -use crate::archive::{Archive, ArchiveWriter, MAX_DECOMPRESSED_SIZE, checked_entry_size}; -use crate::{AsyncContainer, ContainerError, Result, validate_path}; +use crate::archive::{Archive, ArchiveWriter, MAX_DECOMPRESSED_SIZE, UnpackEntry, checked_entry_size}; +use crate::{ContainerError, Result, validate_path}; use lzma_rust2::{XzOptions, XzReader, XzWriter as InnerXzWriter}; use std::io::{Read, Seek, Write}; @@ -23,7 +23,7 @@ impl Archive for Xz { }) } - fn open(source: R, dest: &mut C) -> Result<()> { + fn unpack(source: R, sink: &mut UnpackEntry) -> Result<()> { // `take` bounds how many bytes we decompress from the xz stream, but each tar entry's declared // size is fed to `write_sized`, which pre-allocates from it before reading. Cap the cumulative // declared size too so a header claiming a huge size can't trigger a giant allocation up front. @@ -46,7 +46,7 @@ impl Archive for Xz { let size = checked_entry_size(&mut total_size, entry.size())?; - dest.write_sized_non_blocking(&path, size, &mut |buffer| { + sink(&path, size, &mut |buffer| { entry.read_exact(buffer).map_err(ContainerError::Io)?; Ok(()) })?; diff --git a/document/container/src/archive/zip.rs b/document/container/src/archive/zip.rs index 78118d7a0d1..5a2ea43cbf8 100644 --- a/document/container/src/archive/zip.rs +++ b/document/container/src/archive/zip.rs @@ -1,7 +1,7 @@ //! Zip archive codec. -use crate::archive::{Archive, ArchiveWriter, checked_entry_size}; -use crate::{AsyncContainer, ContainerError, Result, validate_path}; +use crate::archive::{Archive, ArchiveWriter, UnpackEntry, checked_entry_size}; +use crate::{ContainerError, Result, validate_path}; use std::io::{Read, Seek, Write}; use zip::ZipArchive; @@ -24,7 +24,7 @@ impl Archive for Zip { }) } - fn open(source: R, dest: &mut C) -> Result<()> { + fn unpack(source: R, sink: &mut UnpackEntry) -> Result<()> { let mut archive = ZipArchive::new(source).map_err(zip_err)?; // Zip headers declare each entry's uncompressed size up front; `checked_entry_size` caps the running @@ -41,7 +41,7 @@ impl Archive for Zip { let size = checked_entry_size(&mut total_size, entry.size())?; - dest.write_sized_non_blocking(&name, size, &mut |buffer| { + sink(&name, size, &mut |buffer| { entry.read_exact(buffer).map_err(ContainerError::Io)?; Ok(()) })?; diff --git a/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs b/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs index 883aca6f4d0..6f657a11052 100644 --- a/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs +++ b/editor/src/messages/portfolio/document/storage_tests/lifecycle_tests.rs @@ -137,6 +137,49 @@ async fn saving_writes_a_zip_archive_with_the_graphite_extension() { assert_eq!(ArchiveFormat::detect(&content), Some(ArchiveFormat::Zip)); } +#[tokio::test] +async fn opening_a_file_moves_its_resources_into_the_resource_storage() { + use graphene_std::raster::{Color, Image}; + + fn save(editor: &mut EditorTestUtils) -> Vec { + let saved = dispatch(editor, DocumentMessage::SaveDocumentAs); + let content = saved.iter().find_map(|message| match message { + FrontendMessage::TriggerSaveDocument { content, .. } => Some(content.to_vec()), + _ => None, + }); + content.expect("the save is sent to the frontend") + } + async fn saved_resources(content: &[u8]) -> Vec { + let mut unpacked = document_container::backends::memory::MemoryBackend::new(); + document_container::archive::open_auto(content, &mut unpacked).unwrap(); + let mut resources = unpacked.list("resources").await.unwrap_or_default(); + resources.sort(); + resources + } + + let mut editor = EditorTestUtils::create(); + let id = new_document(&mut editor, "Saved"); + editor.create_raster_image(Image::new(2, 2, Color::WHITE), None).await; + let content = save(&mut editor); + let resources = saved_resources(&content).await; + assert!(!resources.is_empty(), "the file carries the document's resources"); + + editor + .handle_message(PortfolioMessage::OpenDocumentFile { + document_name: Some("Opened".into()), + document_path: None, + content, + fresh_identity: true, + }) + .await; + let opened = portfolio(&editor).active_document_id.unwrap(); + assert_ne!(opened, id); + + let container = container(&editor, opened).await; + assert_eq!(container.list("resources").await.unwrap_or_default(), Vec::::new(), "the working copy keeps no resources"); + assert_eq!(saved_resources(&save(&mut editor)).await, resources, "saving writes them from the resource storage"); +} + #[tokio::test] async fn storage_only_container_recovers_and_writes_legacy() { let mut editor = EditorTestUtils::create(); diff --git a/editor/src/messages/portfolio/document_storage_io.rs b/editor/src/messages/portfolio/document_storage_io.rs index 6337efa4bb9..11e8c6bfe29 100644 --- a/editor/src/messages/portfolio/document_storage_io.rs +++ b/editor/src/messages/portfolio/document_storage_io.rs @@ -221,8 +221,19 @@ pub(super) async fn open_document_file( ) -> Message { document_locks::claim(document_id).await; let materialized = async { - let mut container = store.open(document_key(document_id)).await.map_err(|error| error.to_string())?; - document_container::archive::open_auto(&content, &mut container).map_err(|error| error.to_string())?; + let container = store.open(document_key(document_id)).await.map_err(|error| error.to_string())?; + let resources_prefix = format!("{}/", GddV1Layout.resources_dir()); + document_container::archive::unpack_auto(&content, &mut |path, size, fill| { + if path.starts_with(&resources_prefix) { + let mut bytes = vec![0; size]; + fill(&mut bytes)?; + resources.store(&bytes); + return Ok(()); + } + + container.write_sized_non_blocking(path, size, fill) + }) + .map_err(|error| error.to_string())?; Ok::<_, String>(container) } .await; From 1dd119494c6e6fda0f75dde131562d4095ee3524 Mon Sep 17 00:00:00 2001 From: Timon Date: Mon, 16 Nov 2026 00:15:10 +0000 Subject: [PATCH 2/2] Keep resources out of a document's container in the editor --- document/format/src/lib.rs | 9 ++++--- document/format/src/migrate.rs | 10 ++++--- document/format/tests/migration_costs.rs | 12 +++++++++ .../messages/portfolio/document_storage_io.rs | 27 ++++++++++++------- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/document/format/src/lib.rs b/document/format/src/lib.rs index 0c758eb7218..4c0c38da511 100644 --- a/document/format/src/lib.rs +++ b/document/format/src/lib.rs @@ -158,7 +158,7 @@ pub(crate) struct PendingPersist { #[derive(Default)] struct MigrationInputs<'a> { #[cfg(feature = "migrations")] - byte_store: Option<&'a dyn graphene_resource::LoadResource>, + byte_store: Option<&'a dyn ResourceStorage>, #[cfg(feature = "migrations")] host: Option<&'a dyn migration_core::MigrationHost>, #[cfg(not(feature = "migrations"))] @@ -253,7 +253,7 @@ impl Gdd { bytes: &[u8], mut working: AnyContainer, layout: L, - byte_store: Option<&dyn graphene_resource::LoadResource>, + byte_store: Option<&dyn ResourceStorage>, host: Option<&dyn migration_core::MigrationHost>, ) -> Result { document_container::archive::open_auto(bytes, &mut working)?; @@ -276,9 +276,10 @@ impl Gdd { } /// [`open_in`](Self::open_in), offering migrations what only the caller has. `host` gives them the compiler, and without one, those asking for it - /// settle on their fallback for good. `byte_store` holds declarations the container doesn't, such as the editor's global cache. + /// settle on their fallback for good. `byte_store` holds declarations the container doesn't, such as the editor's global cache, + /// and takes the ones migrations introduce in the container's place. #[cfg(feature = "migrations")] - pub async fn open_in_with(working: AnyContainer, layout: L, byte_store: Option<&dyn graphene_resource::LoadResource>, host: Option<&dyn migration_core::MigrationHost>) -> Result { + pub async fn open_in_with(working: AnyContainer, layout: L, byte_store: Option<&dyn ResourceStorage>, host: Option<&dyn migration_core::MigrationHost>) -> Result { Self::open_stored(working, layout, MigrationInputs { byte_store, host }).await } diff --git a/document/format/src/migrate.rs b/document/format/src/migrate.rs index 32836f69c41..ea3703c7b22 100644 --- a/document/format/src/migrate.rs +++ b/document/format/src/migrate.rs @@ -7,7 +7,7 @@ use document_container::AsyncContainer; use document_graph_storage::BlobHash; use document_graph_storage::blob::NewBlobs; use document_graph_storage::{Declarations, Delta, EncodedDeclaration, HistoryMetadata, PeerId, Registry, ResourceEntry, ResourceId, Rev, Session, Subject, TimeStamp}; -use graphene_resource::LoadResource; +use graphene_resource::ResourceStorage; use migration_core::{Declaration, DeclarationResolver, MigrationHost, NodeMigration, Payload, PayloadCodec}; use migration_runner::{DocumentPayloads, HistoryOutcome, MigrationBlobs, RunnerError, holds, migrate_history, migrate_registry, references, registry_at, run_format_migrations}; @@ -70,7 +70,7 @@ fn payload_codec(codec: Codec) -> PayloadCodec { impl Gdd { /// Rewrite history through every registered node migration whose source declaration it references, in dependency order. Returns whether any ran. - pub(crate) async fn migrate_nodes(&mut self, byte_store: Option<&dyn LoadResource>, host: Option<&dyn MigrationHost>) -> Result { + pub(crate) async fn migrate_nodes(&mut self, byte_store: Option<&dyn ResourceStorage>, host: Option<&dyn MigrationHost>) -> Result { let migrations = migration_core::registered_migrations(); if migrations.is_empty() { return Ok(false); @@ -112,6 +112,10 @@ impl Gdd { // Declarations the migrations introduced need their bytes next to the other resources let referenced = self.session.all_declaration_resources(); for declaration in resolver.encoded.values().filter(|declaration| referenced.contains_key(&declaration.id)) { + if let Some(byte_store) = byte_store { + byte_store.store(&declaration.bytes); + continue; + } let path = self.layout.resource_path(&declaration.hash); if !self.working.exists(&path).await { self.working.write_non_blocking(&path, &declaration.bytes)?; @@ -124,7 +128,7 @@ impl Gdd { } /// Every declaration this document's nodes and history reference, from `byte_store` where it has them and otherwise from the container. - async fn load_stored_declarations(&self, byte_store: Option<&dyn LoadResource>) -> Declarations { + async fn load_stored_declarations(&self, byte_store: Option<&dyn ResourceStorage>) -> Declarations { let mut declarations = Declarations::new(); for (id, hash) in self.session.all_declaration_resources() { diff --git a/document/format/tests/migration_costs.rs b/document/format/tests/migration_costs.rs index 9c689687d1f..33447c1715e 100644 --- a/document/format/tests/migration_costs.rs +++ b/document/format/tests/migration_costs.rs @@ -73,6 +73,18 @@ impl LoadResource for EmptyStore { } } +impl ResourceStorage for EmptyStore { + fn store(&self, data: &[u8]) -> ResourceHash { + ResourceHash::from(data) + } + + fn contains(&self, _hash: &ResourceHash) -> bool { + false + } + + fn garbage_collect(&self, _used: &[ResourceHash]) {} +} + fn network_of(nodes: &[(u64, &'static str)]) -> NodeNetwork { let node = |identifier: &'static str| DocumentNode { implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new(identifier)), diff --git a/editor/src/messages/portfolio/document_storage_io.rs b/editor/src/messages/portfolio/document_storage_io.rs index 11e8c6bfe29..b07de0cbc06 100644 --- a/editor/src/messages/portfolio/document_storage_io.rs +++ b/editor/src/messages/portfolio/document_storage_io.rs @@ -20,7 +20,6 @@ use document_container::store::{DocumentKey, DocumentStore, MemoryStore}; use document_container::{AnyContainer, AsyncContainer}; use document_format::{Error as DocumentFormatError, GddV1, GddV1Layout, Layout, ReadError}; use document_graph_storage::{Declarations, PeerId, UserId}; -use graph_craft::application_io::resource::LoadResource; use graph_craft::application_io::resource::{DataSource, ResourceHash, ResourceStorage}; use std::collections::HashSet; use std::path::PathBuf; @@ -298,12 +297,12 @@ pub(super) async fn load_from_container(container: AnyContainer, document_id: Do } /// Open a document's storage, offering its migrations the compiler and the global cache's declarations. -async fn open_migrating(container: AnyContainer, resources: &dyn LoadResource) -> Result { +async fn open_migrating(container: AnyContainer, resources: &dyn ResourceStorage) -> Result { let host = migration_compiled::CompilerHost::default(); GddV1::open_in_with(container, GddV1Layout, Some(resources), Some(&host)).await } -async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id: DocumentId, resources: &dyn LoadResource) -> Option { +async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id: DocumentId, resources: &dyn ResourceStorage) -> Option { match open_migrating(container.clone(), resources).await { Ok(gdd) => return Some(gdd), Err(DocumentFormatError::MissingManifest) => {} @@ -338,12 +337,12 @@ async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id async fn load_declarations(gdd: Option<&GddV1>, resources: &ResourcesHandle, document_id: DocumentId) -> Declarations { let Some(gdd) = gdd else { return Declarations::new() }; - // Migrations store the declarations they introduce in the container, which this device's cache may not have seen + // A container written before resources stayed out of it still holds some hydrate_resources(gdd, resources, document_id).await; gdd.declarations(resources).await } -/// Copy every resource stored in `gdd`'s container that the global cache lacks into it. +/// Move every resource stored in `gdd`'s container into the global cache. async fn hydrate_resources(gdd: &GddV1, byte_store: &ResourcesHandle, document_id: DocumentId) { let hashes = match gdd.resource_hashes().await { Ok(hashes) => hashes, @@ -353,12 +352,20 @@ async fn hydrate_resources(gdd: &GddV1, byte_store: &ResourcesHandle, document_i } }; - for hash in hashes.into_iter().filter(|hash| !byte_store.contains(hash)) { - match gdd.read_resource(&hash).await { - Ok(holder) => { - byte_store.store(holder.as_slice()); + for hash in hashes { + if !byte_store.contains(&hash) { + match gdd.read_resource(&hash).await { + Ok(holder) => { + byte_store.store(holder.as_slice()); + } + Err(error) => { + log::error!("Opening document {document_id:?}: failed to read resource {hash}: {error}"); + continue; + } } - Err(error) => log::error!("Opening document {document_id:?}: failed to read resource {hash}: {error}"), + } + if let Err(error) = gdd.remove_resource(&hash) { + log::error!("Opening document {document_id:?}: failed to remove resource {hash}: {error}"); } } }