Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions document/container/src/archive.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,18 @@ pub trait Archive {

fn writer<W: Write + Seek>(output: W) -> Result<Self::Writer<W>>;

fn unpack<R: Read + Seek>(source: R, sink: &mut UnpackEntry) -> Result<()>;

/// Read entries from `source` and write each into `dest`, streaming so neither the full
/// archive nor the full container ever sits in memory at once.
fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()>;
fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()> {
Self::unpack(source, &mut |path, size, fill| dest.write_sized_non_blocking(path, size, fill))
}
}

/// See [`Archive::unpack`].
pub type UnpackEntry<'a> = dyn FnMut(&str, usize, &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> + 'a;

pub trait ArchiveWriter: Sized {
type Sink;

Expand Down Expand Up @@ -100,12 +107,18 @@ impl ArchiveFormat {
/// Errors if the bytes are neither a recognized xz nor zip archive.
#[cfg(any(feature = "xz", feature = "zip"))]
pub fn open_auto<C: AsyncContainer>(bytes: &[u8], dest: &mut C) -> Result<()> {
unpack_auto(bytes, &mut |path, size, fill| dest.write_sized_non_blocking(path, size, fill))
}

/// Like [`open_auto`], but hands each entry to `sink` instead of writing it into a container.
#[cfg(any(feature = "xz", feature = "zip"))]
pub fn unpack_auto(bytes: &[u8], sink: &mut UnpackEntry) -> Result<()> {
let source = std::io::Cursor::new(bytes);
match ArchiveFormat::detect(bytes) {
#[cfg(feature = "xz")]
Some(ArchiveFormat::Xz) => Xz::open(source, dest),
Some(ArchiveFormat::Xz) => Xz::unpack(source, sink),
#[cfg(feature = "zip")]
Some(ArchiveFormat::Zip) => Zip::open(source, dest),
Some(ArchiveFormat::Zip) => Zip::unpack(source, sink),
#[allow(unreachable_patterns)]
Some(format) => Err(ContainerError::Codec(format!("tried to open {format:?}, but the binary was compiled without the feature enabled "))),
None => Err(ContainerError::Codec("unrecognized archive format (not xz or zip) ".into())),
Expand Down
8 changes: 4 additions & 4 deletions document/container/src/archive/xz.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Xz-compressed tarball archive codec.

use crate::archive::{Archive, ArchiveWriter, MAX_DECOMPRESSED_SIZE, checked_entry_size};
use crate::{AsyncContainer, ContainerError, Result, validate_path};
use crate::archive::{Archive, ArchiveWriter, MAX_DECOMPRESSED_SIZE, UnpackEntry, checked_entry_size};
use crate::{ContainerError, Result, validate_path};
use lzma_rust2::{XzOptions, XzReader, XzWriter as InnerXzWriter};
use std::io::{Read, Seek, Write};

Expand All @@ -23,7 +23,7 @@ impl Archive for Xz {
})
}

fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()> {
fn unpack<R: Read + Seek>(source: R, sink: &mut UnpackEntry) -> Result<()> {
// `take` bounds how many bytes we decompress from the xz stream, but each tar entry's declared
// size is fed to `write_sized`, which pre-allocates from it before reading. Cap the cumulative
// declared size too so a header claiming a huge size can't trigger a giant allocation up front.
Expand All @@ -46,7 +46,7 @@ impl Archive for Xz {

let size = checked_entry_size(&mut total_size, entry.size())?;

dest.write_sized_non_blocking(&path, size, &mut |buffer| {
sink(&path, size, &mut |buffer| {
entry.read_exact(buffer).map_err(ContainerError::Io)?;
Ok(())
})?;
Expand Down
8 changes: 4 additions & 4 deletions document/container/src/archive/zip.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Zip archive codec.

use crate::archive::{Archive, ArchiveWriter, checked_entry_size};
use crate::{AsyncContainer, ContainerError, Result, validate_path};
use crate::archive::{Archive, ArchiveWriter, UnpackEntry, checked_entry_size};
use crate::{ContainerError, Result, validate_path};
use std::io::{Read, Seek, Write};

use zip::ZipArchive;
Expand All @@ -24,7 +24,7 @@ impl Archive for Zip {
})
}

fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()> {
fn unpack<R: Read + Seek>(source: R, sink: &mut UnpackEntry) -> Result<()> {
let mut archive = ZipArchive::new(source).map_err(zip_err)?;

// Zip headers declare each entry's uncompressed size up front; `checked_entry_size` caps the running
Expand All @@ -41,7 +41,7 @@ impl Archive for Zip {

let size = checked_entry_size(&mut total_size, entry.size())?;

dest.write_sized_non_blocking(&name, size, &mut |buffer| {
sink(&name, size, &mut |buffer| {
entry.read_exact(buffer).map_err(ContainerError::Io)?;
Ok(())
})?;
Expand Down
9 changes: 5 additions & 4 deletions document/format/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ pub(crate) struct PendingPersist {
#[derive(Default)]
struct MigrationInputs<'a> {
#[cfg(feature = "migrations")]
byte_store: Option<&'a dyn graphene_resource::LoadResource>,
byte_store: Option<&'a dyn ResourceStorage>,
#[cfg(feature = "migrations")]
host: Option<&'a dyn migration_core::MigrationHost>,
#[cfg(not(feature = "migrations"))]
Expand Down Expand Up @@ -253,7 +253,7 @@ impl<L: Layout> Gdd<L> {
bytes: &[u8],
mut working: AnyContainer,
layout: L,
byte_store: Option<&dyn graphene_resource::LoadResource>,
byte_store: Option<&dyn ResourceStorage>,
host: Option<&dyn migration_core::MigrationHost>,
) -> Result<Self, Error> {
document_container::archive::open_auto(bytes, &mut working)?;
Expand All @@ -276,9 +276,10 @@ impl<L: Layout> Gdd<L> {
}

/// [`open_in`](Self::open_in), offering migrations what only the caller has. `host` gives them the compiler, and without one, those asking for it
/// settle on their fallback for good. `byte_store` holds declarations the container doesn't, such as the editor's global cache.
/// settle on their fallback for good. `byte_store` holds declarations the container doesn't, such as the editor's global cache,
/// and takes the ones migrations introduce in the container's place.
#[cfg(feature = "migrations")]
pub async fn open_in_with(working: AnyContainer, layout: L, byte_store: Option<&dyn graphene_resource::LoadResource>, host: Option<&dyn migration_core::MigrationHost>) -> Result<Self, Error> {
pub async fn open_in_with(working: AnyContainer, layout: L, byte_store: Option<&dyn ResourceStorage>, host: Option<&dyn migration_core::MigrationHost>) -> Result<Self, Error> {
Self::open_stored(working, layout, MigrationInputs { byte_store, host }).await
}

Expand Down
10 changes: 7 additions & 3 deletions document/format/src/migrate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use document_container::AsyncContainer;
use document_graph_storage::BlobHash;
use document_graph_storage::blob::NewBlobs;
use document_graph_storage::{Declarations, Delta, EncodedDeclaration, HistoryMetadata, PeerId, Registry, ResourceEntry, ResourceId, Rev, Session, Subject, TimeStamp};
use graphene_resource::LoadResource;
use graphene_resource::ResourceStorage;
use migration_core::{Declaration, DeclarationResolver, MigrationHost, NodeMigration, Payload, PayloadCodec};
use migration_runner::{DocumentPayloads, HistoryOutcome, MigrationBlobs, RunnerError, holds, migrate_history, migrate_registry, references, registry_at, run_format_migrations};

Expand Down Expand Up @@ -70,7 +70,7 @@ fn payload_codec(codec: Codec) -> PayloadCodec {

impl<L: Layout> Gdd<L> {
/// Rewrite history through every registered node migration whose source declaration it references, in dependency order. Returns whether any ran.
pub(crate) async fn migrate_nodes(&mut self, byte_store: Option<&dyn LoadResource>, host: Option<&dyn MigrationHost>) -> Result<bool, Error> {
pub(crate) async fn migrate_nodes(&mut self, byte_store: Option<&dyn ResourceStorage>, host: Option<&dyn MigrationHost>) -> Result<bool, Error> {
let migrations = migration_core::registered_migrations();
if migrations.is_empty() {
return Ok(false);
Expand Down Expand Up @@ -112,6 +112,10 @@ impl<L: Layout> Gdd<L> {
// Declarations the migrations introduced need their bytes next to the other resources
let referenced = self.session.all_declaration_resources();
for declaration in resolver.encoded.values().filter(|declaration| referenced.contains_key(&declaration.id)) {
if let Some(byte_store) = byte_store {
byte_store.store(&declaration.bytes);
continue;
}
let path = self.layout.resource_path(&declaration.hash);
if !self.working.exists(&path).await {
self.working.write_non_blocking(&path, &declaration.bytes)?;
Expand All @@ -124,7 +128,7 @@ impl<L: Layout> Gdd<L> {
}

/// Every declaration this document's nodes and history reference, from `byte_store` where it has them and otherwise from the container.
async fn load_stored_declarations(&self, byte_store: Option<&dyn LoadResource>) -> Declarations {
async fn load_stored_declarations(&self, byte_store: Option<&dyn ResourceStorage>) -> Declarations {
let mut declarations = Declarations::new();

for (id, hash) in self.session.all_declaration_resources() {
Expand Down
12 changes: 12 additions & 0 deletions document/format/tests/migration_costs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,18 @@ impl LoadResource for EmptyStore {
}
}

impl ResourceStorage for EmptyStore {
fn store(&self, data: &[u8]) -> ResourceHash {
ResourceHash::from(data)
}

fn contains(&self, _hash: &ResourceHash) -> bool {
false
}

fn garbage_collect(&self, _used: &[ResourceHash]) {}
}

fn network_of(nodes: &[(u64, &'static str)]) -> NodeNetwork {
let node = |identifier: &'static str| DocumentNode {
implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new(identifier)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,49 @@ async fn saving_writes_a_zip_archive_with_the_graphite_extension() {
assert_eq!(ArchiveFormat::detect(&content), Some(ArchiveFormat::Zip));
}

#[tokio::test]
async fn opening_a_file_moves_its_resources_into_the_resource_storage() {
use graphene_std::raster::{Color, Image};

fn save(editor: &mut EditorTestUtils) -> Vec<u8> {
let saved = dispatch(editor, DocumentMessage::SaveDocumentAs);
let content = saved.iter().find_map(|message| match message {
FrontendMessage::TriggerSaveDocument { content, .. } => Some(content.to_vec()),
_ => None,
});
content.expect("the save is sent to the frontend")
}
async fn saved_resources(content: &[u8]) -> Vec<String> {
let mut unpacked = document_container::backends::memory::MemoryBackend::new();
document_container::archive::open_auto(content, &mut unpacked).unwrap();
let mut resources = unpacked.list("resources").await.unwrap_or_default();
resources.sort();
resources
}

let mut editor = EditorTestUtils::create();
let id = new_document(&mut editor, "Saved");
editor.create_raster_image(Image::new(2, 2, Color::WHITE), None).await;
let content = save(&mut editor);
let resources = saved_resources(&content).await;
assert!(!resources.is_empty(), "the file carries the document's resources");

editor
.handle_message(PortfolioMessage::OpenDocumentFile {
document_name: Some("Opened".into()),
document_path: None,
content,
fresh_identity: true,
})
.await;
let opened = portfolio(&editor).active_document_id.unwrap();
assert_ne!(opened, id);

let container = container(&editor, opened).await;
assert_eq!(container.list("resources").await.unwrap_or_default(), Vec::<String>::new(), "the working copy keeps no resources");
assert_eq!(saved_resources(&save(&mut editor)).await, resources, "saving writes them from the resource storage");
}

#[tokio::test]
async fn storage_only_container_recovers_and_writes_legacy() {
let mut editor = EditorTestUtils::create();
Expand Down
42 changes: 30 additions & 12 deletions editor/src/messages/portfolio/document_storage_io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ use document_container::store::{DocumentKey, DocumentStore, MemoryStore};
use document_container::{AnyContainer, AsyncContainer};
use document_format::{Error as DocumentFormatError, GddV1, GddV1Layout, Layout, ReadError};
use document_graph_storage::{Declarations, PeerId, UserId};
use graph_craft::application_io::resource::LoadResource;
use graph_craft::application_io::resource::{DataSource, ResourceHash, ResourceStorage};
use std::collections::HashSet;
use std::path::PathBuf;
Expand Down Expand Up @@ -221,8 +220,19 @@ pub(super) async fn open_document_file(
) -> Message {
document_locks::claim(document_id).await;
let materialized = async {
let mut container = store.open(document_key(document_id)).await.map_err(|error| error.to_string())?;
document_container::archive::open_auto(&content, &mut container).map_err(|error| error.to_string())?;
let container = store.open(document_key(document_id)).await.map_err(|error| error.to_string())?;
let resources_prefix = format!("{}/", GddV1Layout.resources_dir());
document_container::archive::unpack_auto(&content, &mut |path, size, fill| {
if path.starts_with(&resources_prefix) {
let mut bytes = vec![0; size];
fill(&mut bytes)?;
resources.store(&bytes);
return Ok(());
}

container.write_sized_non_blocking(path, size, fill)
})
.map_err(|error| error.to_string())?;
Ok::<_, String>(container)
}
.await;
Expand Down Expand Up @@ -287,12 +297,12 @@ pub(super) async fn load_from_container(container: AnyContainer, document_id: Do
}

/// Open a document's storage, offering its migrations the compiler and the global cache's declarations.
async fn open_migrating(container: AnyContainer, resources: &dyn LoadResource) -> Result<GddV1, DocumentFormatError> {
async fn open_migrating(container: AnyContainer, resources: &dyn ResourceStorage) -> Result<GddV1, DocumentFormatError> {
let host = migration_compiled::CompilerHost::default();
GddV1::open_in_with(container, GddV1Layout, Some(resources), Some(&host)).await
}

async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id: DocumentId, resources: &dyn LoadResource) -> Option<GddV1> {
async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id: DocumentId, resources: &dyn ResourceStorage) -> Option<GddV1> {
match open_migrating(container.clone(), resources).await {
Ok(gdd) => return Some(gdd),
Err(DocumentFormatError::MissingManifest) => {}
Expand Down Expand Up @@ -327,12 +337,12 @@ async fn open_storage_or_fallback_to_legacy(container: AnyContainer, document_id
async fn load_declarations(gdd: Option<&GddV1>, resources: &ResourcesHandle, document_id: DocumentId) -> Declarations {
let Some(gdd) = gdd else { return Declarations::new() };

// Migrations store the declarations they introduce in the container, which this device's cache may not have seen
// A container written before resources stayed out of it still holds some
hydrate_resources(gdd, resources, document_id).await;
gdd.declarations(resources).await
}

/// Copy every resource stored in `gdd`'s container that the global cache lacks into it.
/// Move every resource stored in `gdd`'s container into the global cache.
async fn hydrate_resources(gdd: &GddV1, byte_store: &ResourcesHandle, document_id: DocumentId) {
let hashes = match gdd.resource_hashes().await {
Ok(hashes) => hashes,
Expand All @@ -342,12 +352,20 @@ async fn hydrate_resources(gdd: &GddV1, byte_store: &ResourcesHandle, document_i
}
};

for hash in hashes.into_iter().filter(|hash| !byte_store.contains(hash)) {
match gdd.read_resource(&hash).await {
Ok(holder) => {
byte_store.store(holder.as_slice());
for hash in hashes {
if !byte_store.contains(&hash) {
match gdd.read_resource(&hash).await {
Ok(holder) => {
byte_store.store(holder.as_slice());
}
Err(error) => {
log::error!("Opening document {document_id:?}: failed to read resource {hash}: {error}");
continue;
}
}
Err(error) => log::error!("Opening document {document_id:?}: failed to read resource {hash}: {error}"),
}
if let Err(error) = gdd.remove_resource(&hash) {
log::error!("Opening document {document_id:?}: failed to remove resource {hash}: {error}");
}
}
}
Expand Down
Loading